Reorganising the repository - external renames and moves (#4074)

* Adding first rough ouline of the repository structure * Remove old CI stuff * add title * formatting fixes * move node-exits job's script to scripts dir * Move docs into subdir * move to bin * move maintainence scripts, configs and helpers into its own dir * add .local to ignore * move core->client * start up 'test' area * move test client * move test runtime * make test move compile * Add dependencies rule enforcement. * Fix indexing. * Update docs to reflect latest changes * Moving /srml->/paint * update docs * move client/sr-* -> primitives/ * clean old readme * remove old broken code in rhd * update lock * Step 1. * starting to untangle client * Fix after merge. * start splitting out client interfaces * move children and blockchain interfaces * Move trie and state-machine to primitives. * Fix WASM builds. * fixing broken imports * more interface moves * move backend and light to interfaces * move CallExecutor * move cli off client * moving around more interfaces * re-add consensus crates into the mix * fix subkey path * relieve client from executor * starting to pull out client from grandpa * move is_decendent_of out of client * grandpa still depends on client directly * lemme tests pass * rename srml->paint * Make it compile. * rename interfaces->client-api * Move keyring to primitives. * fixup libp2p dep * fix broken use * allow dependency enforcement to fail * move fork-tree * Moving wasm-builder * make env * move build-script-utils * fixup broken crate depdencies and names * fix imports for authority discovery * fix typo * update cargo.lock * fixing imports * Fix paths and add missing crates * re-add missing crates
2026-05-30 03:31:05 +00:00 · 2019-11-14 21:51:17 +01:00
parent becc3b0a4f
commit 60e5011c72
809 changed files with 7801 additions and 6464 deletions
@@ -0,0 +1,55 @@
+# Note: We don't use Alpine and its packaged Rust/Cargo because they're too often out of date,
+# preventing them from being used to build Substrate/Polkadot.
+
+FROM phusion/baseimage:0.10.2 as builder
+LABEL maintainer="chevdor@gmail.com"
+LABEL description="This is the build stage for Substrate. Here we create the binary."
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+ARG PROFILE=release
+WORKDIR /substrate
+
+COPY . /substrate
+
+RUN apt-get update && \
+	apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \
+	apt-get install -y cmake pkg-config libssl-dev git clang
+
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \
+	export PATH="$PATH:$HOME/.cargo/bin" && \
+	rustup toolchain install nightly && \
+	rustup target add wasm32-unknown-unknown --toolchain nightly && \
+	rustup default nightly && \
+	rustup default stable && \
+	cargo build "--$PROFILE"
+
+# ===== SECOND STAGE ======
+
+FROM phusion/baseimage:0.10.2
+LABEL maintainer="chevdor@gmail.com"
+LABEL description="This is the 2nd stage: a very small image where we copy the Substrate binary."
+ARG PROFILE=release
+
+RUN mv /usr/share/ca* /tmp && \
+	rm -rf /usr/share/*  && \
+	mv /tmp/ca-certificates /usr/share/ && \
+	mkdir -p /root/.local/share/Polkadot && \
+	ln -s /root/.local/share/Polkadot /data && \
+	useradd -m -u 1000 -U -s /bin/sh -d /substrate substrate
+
+COPY --from=builder /substrate/target/$PROFILE/substrate /usr/local/bin
+
+# checks
+RUN ldd /usr/local/bin/substrate && \
+	/usr/local/bin/substrate --version
+
+# Shrinking
+RUN rm -rf /usr/lib/python* && \
+	rm -rf /usr/bin /usr/sbin /usr/share/man
+
+USER substrate
+EXPOSE 30333 9933 9944
+VOLUME ["/data"]
+
+CMD ["/usr/local/bin/substrate"]
@@ -0,0 +1,29 @@
+#!/usr/bin/env sh
+
+# Script for building only the WASM binary of the given project.
+
+set -e
+
+PROJECT_ROOT=`git rev-parse --show-toplevel`
+
+if [ "$#" -lt 1 ]; then
+  echo "You need to pass the name of the crate you want to compile!"
+  exit 1
+fi
+
+WASM_BUILDER_RUNNER="$PROJECT_ROOT/target/release/wbuild-runner/$1"
+
+if [ -z "$2" ]; then
+  export WASM_TARGET_DIRECTORY=$(pwd)
+else
+  export WASM_TARGET_DIRECTORY=$2
+fi
+
+if [ -d $WASM_BUILDER_RUNNER ]; then
+  export DEBUG=false
+  export OUT_DIR="$PROJECT_ROOT/target/release/build"
+  cargo run --release --manifest-path="$WASM_BUILDER_RUNNER/Cargo.toml" \
+    | grep -vE "cargo:rerun-if-|Executing build command"
+else
+  cargo build --release -p $1
+fi
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# Script that checks that a node exits after `SIGINT` was send.
+
+set -e
+
+cargo build --release
+./target/release/substrate --dev &
+PID=$!
+
+# Let the chain running for 60 seconds
+sleep 60
+
+# Send `SIGINT` and give the process 30 seconds to end
+kill -INT $PID
+timeout 30 tail --pid=$PID -f /dev/null
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+ROOT=`dirname "$0"`
+
+# A list of directories which contain wasm projects.
+SRCS=(
+	"core/executor/wasm"
+	"node/runtime/wasm"
+	"node-template/runtime/wasm"
+	"core/test-runtime/wasm"
+)
+
+# Make pushd/popd silent.
+
+pushd () {
+	command pushd "$@" > /dev/null
+}
+
+popd () {
+	command popd "$@" > /dev/null
+}
@@ -0,0 +1,31 @@
+FROM debian:stretch-slim
+
+# metadata
+ARG VCS_REF
+ARG BUILD_DATE
+
+LABEL io.parity.image.authors="devops-team@parity.io" \
+	io.parity.image.vendor="Parity Technologies" \
+	io.parity.image.title="parity/subkey" \
+	io.parity.image.description="Subkey: key generating utility for Substrate." \
+	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/.maintain/docker/subkey.Dockerfile" \
+	io.parity.image.revision="${VCS_REF}" \
+	io.parity.image.created="${BUILD_DATE}" \
+	io.parity.image.documentation="https://github.com/paritytech/substrate/tree/${VCS_REF}/subkey"
+
+# show backtraces
+ENV RUST_BACKTRACE 1
+
+# add user
+RUN useradd -m -u 1000 -U -s /bin/sh -d /subkey subkey
+
+# add subkey binary to docker image
+COPY ./subkey /usr/local/bin
+
+USER subkey
+
+# check if executable works in this container
+RUN /usr/local/bin/subkey --version
+
+ENTRYPOINT ["/usr/local/bin/subkey"]
+
@@ -0,0 +1,45 @@
+FROM debian:stretch-slim
+
+# metadata
+ARG VCS_REF
+ARG BUILD_DATE
+
+LABEL io.parity.image.authors="devops-team@parity.io" \
+	io.parity.image.vendor="Parity Technologies" \
+	io.parity.image.title="parity/substrate" \
+	io.parity.image.description="Substrate: The platform for blockchain innovators." \
+	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/.maintain/docker/Dockerfile" \
+	io.parity.image.revision="${VCS_REF}" \
+	io.parity.image.created="${BUILD_DATE}" \
+	io.parity.image.documentation="https://wiki.parity.io/Parity-Substrate"
+
+# show backtraces
+ENV RUST_BACKTRACE 1
+
+# install tools and dependencies
+RUN apt-get update && \
+	DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \
+	DEBIAN_FRONTEND=noninteractive apt-get install -y \
+		libssl1.1 \
+		ca-certificates \
+		curl && \
+# apt cleanup
+	apt-get autoremove -y && \
+	apt-get clean && \
+	find /var/lib/apt/lists/ -type f -not -name lock -delete; \
+# add user
+	useradd -m -u 1000 -U -s /bin/sh -d /substrate substrate
+
+# add substrate binary to docker image
+COPY ./substrate /usr/local/bin
+
+USER substrate
+
+# check if executable works in this container
+RUN /usr/local/bin/substrate --version
+
+EXPOSE 30333 9933 9944
+VOLUME ["/substrate"]
+
+ENTRYPOINT ["/usr/local/bin/substrate"]
+
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# The script is meant to check if the rules regarding packages
+# dependencies are satisfied.
+# The general format is:
+# [top-lvl-dir]<[crate-name-prefix]
+
+# For instance no crate within `./client` directory
+# is allowed to import any crate with a directory path containing `paint`.
+# Such rule is just: `client<paint`.
+
+# The script should be run from the main repo directory!
+
+set -u
+
+PLEASE_DONT=(
+	"client<paint"
+	"client<node"
+	"paint<node"
+	"paint<client"
+	"primitives<paint"
+	"primitives<client"
+)
+
+VIOLATIONS=()
+PACKAGES=()
+
+for rule in "${PLEASE_DONT[@]}"
+do
+	from=$(echo $rule | cut -f1 -d\<)
+	to=$(echo $rule | cut -f2 -d\<)
+
+	cd $from
+	echo "Checking rule $rule"
+	packages=$(find -name Cargo.toml | xargs grep -wn "path.*$to")
+	has_references=$(echo -n $packages | wc -c)
+	if [ "$has_references" != "0" ]; then
+		VIOLATIONS+=("$rule")
+		# Find packages that violate:
+		PACKAGES+=("$packages")
+	fi
+	cd - > /dev/null
+done
+
+# Display violations and fail
+I=0
+for v in "${VIOLATIONS[@]}"
+do
+	cat << EOF
+
+===========================================
+======= Violation of rule: $v
+===========================================
+${PACKAGES[$I]}
+
+
+EOF
+	I=$I+1
+done
+
+exit ${#VIOLATIONS[@]}
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+RETRY_COUNT=10
+RETRY_ATTEMPT=0
+SLEEP_TIME=15
+TARGET_HOST="$1"
+COMMIT=$(cat artifacts/substrate/VERSION)
+DOWNLOAD_URL="https://releases.parity.io/substrate/x86_64-debian:stretch/${COMMIT}/substrate"
+POST_DATA='{"extra_vars":{"artifact_path":"'${DOWNLOAD_URL}'","target_host":"'${TARGET_HOST}'"}}'
+
+JOB_ID=$(wget -O - --header "Authorization: Bearer ${AWX_TOKEN}" --header "Content-type: application/json" --post-data "${POST_DATA}" https://ansible-awx.parity.io/api/v2/job_templates/32/launch/ | jq .job)
+
+echo "Launched job: $JOB_ID"
+
+
+while [ ${RETRY_ATTEMPT} -le ${RETRY_COUNT} ] ; do
+	export RETRY_RESULT=$(wget -O - --header "Authorization: Bearer ${AWX_TOKEN}"  https://ansible-awx.parity.io/api/v2/jobs/${JOB_ID}/ | jq .status)
+	RETRY_ATTEMPT=$(( $RETRY_ATTEMPT +1 ))
+	sleep $SLEEP_TIME
+	if [ $(echo $RETRY_RESULT | egrep  -e successful -e failed) ] ; then
+            break
+        fi
+done
+
+AWX_OUTPUT=$(wget -O - --header "Authorization: Bearer ${AWX_TOKEN}"  https://ansible-awx.parity.io/api/v2/jobs/${JOB_ID}/stdout?format=txt_download)
+
+echo "AWX job log:"
+echo "${AWX_OUTPUT}"
+
+
+JOB_STATUS=$(wget -O - --header "Authorization: Bearer ${AWX_TOKEN}"  https://ansible-awx.parity.io/api/v2/jobs/${JOB_ID}/ | jq .status )
+
+echo "==================================="
+echo -e "Ansible AWX Remote Job: ${JOB_ID} \x1B[31mStatus: ${JOB_STATUS}\x1B[0m"
+echo "==================================="
@@ -0,0 +1,6 @@
+/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+brew install openssl cmake
+curl https://sh.rustup.rs -sSf | sh
+source ~/.cargo/env
+cargo install --git https://github.com/paritytech/substrate subkey
+cargo install --git https://github.com/paritytech/substrate substrate
@@ -0,0 +1,52 @@
+#!/bin/sh
+#
+# check if line width of rust source files is not beyond x characters
+#
+
+
+BASE_BRANCH="origin/master"
+LINE_WIDTH="121"
+GOOD_LINE_WIDTH="101"
+
+
+git diff --name-only ${BASE_BRANCH}...${CI_COMMIT_SHA} \*.rs | ( while read file
+do
+  if [ ! -f ${file} ];
+  then
+	echo "Skipping removed file."
+  elif git diff ${BASE_BRANCH}...${CI_COMMIT_SHA} ${file} | grep -q "^+.\{${LINE_WIDTH}\}"
+  then
+    if [ -z "${FAIL}" ]
+    then
+      echo "| warning!"
+      echo "| Lines should not be longer than 120 characters."
+      echo "| "
+      echo "| see more https://wiki.parity.io/Substrate-Style-Guide"
+      echo "|"
+      FAIL="true"
+    fi
+    echo "| file: ${file}"
+    git diff ${BASE_BRANCH}...${CI_COMMIT_SHA} ${file} \
+      | grep -n "^+.\{${LINE_WIDTH}\}"
+    echo "|"
+  else
+    if git diff ${BASE_BRANCH}...${CI_COMMIT_SHA} ${file} | grep -q "^+.\{${GOOD_LINE_WIDTH}\}"
+    then
+      if [ -z "${FAIL}" ]
+      then
+        echo "| warning!"
+        echo "| Lines should be longer than 100 characters only in exceptional circumstances!"
+        echo "| "
+        echo "| see more https://wiki.parity.io/Substrate-Style-Guide"
+        echo "|"
+      fi
+      echo "| file: ${file}"
+      git diff ${BASE_BRANCH}...${CI_COMMIT_SHA} ${file} \
+        | grep -n "^+.\{${LINE_WIDTH}\}"
+      echo "|"
+    fi
+  fi
+done
+
+test -z "${FAIL}"
+)
@@ -0,0 +1,119 @@
+#!/bin/sh
+#
+#
+# check for any changes in the node/src/runtime, paint/ and core/sr_* trees. if
+# there are any changes found, it should mark the PR breaksconsensus and
+# "auto-fail" the PR if there isn't a change in the runtime/src/lib.rs file 
+# that alters the version.
+
+set -e # fail on any error
+
+
+# give some context
+git log --graph --oneline --decorate=short -n 10
+
+
+RUNTIME="node/runtime/wasm/target/wasm32-unknown-unknown/release/node_runtime.compact.wasm"
+VERSIONS_FILE="node/runtime/src/lib.rs"
+
+github_label () {
+	echo
+	echo "# run github-api job for labeling it ${1}"
+	curl -sS -X POST \
+		-F "token=${CI_JOB_TOKEN}" \
+		-F "ref=master" \
+		-F "variables[LABEL]=${1}" \
+		-F "variables[PRNO]=${CI_COMMIT_REF_NAME}" \
+		${GITLAB_API}/projects/${GITHUB_API_PROJECT}/trigger/pipeline
+}
+
+
+
+
+# check if the wasm sources changed
+if ! git diff --name-only origin/master...${CI_COMMIT_SHA} \
+	| grep -q -e '^node/src/runtime' -e '^paint/' -e '^core/sr-' | grep -v -e '^core/sr-arithmetic/fuzzer'
+then
+	cat <<-EOT
+	
+	no changes to the runtime source code detected
+
+	EOT
+
+	exit 0
+fi
+
+
+
+# check for spec_version updates: if the spec versions changed, then there is
+# consensus-critical logic that has changed. the runtime wasm blobs must be
+# rebuilt.
+
+add_spec_version="$(git diff origin/master...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+	| sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
+sub_spec_version="$(git diff origin/master...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+	| sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
+
+
+# see if the version and the binary blob changed
+if [ "${add_spec_version}" != "${sub_spec_version}" ]
+then
+
+	github_label "B2-breaksapi"
+
+	cat <<-EOT
+		
+		changes to the runtime sources and changes in the spec version.
+	
+		spec_version: ${sub_spec_version} -> ${add_spec_version}
+	
+	EOT
+	exit 0
+
+else
+	# check for impl_version updates: if only the impl versions changed, we assume
+	# there is no consensus-critical logic that has changed.
+
+	add_impl_version="$(git diff origin/master...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+		| sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
+	sub_impl_version="$(git diff origin/master...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+		| sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
+
+
+	# see if the impl version changed
+	if [ "${add_impl_version}" != "${sub_impl_version}" ]
+	then
+		cat <<-EOT
+		
+		changes to the runtime sources and changes in the impl version.
+
+		impl_version: ${sub_impl_version} -> ${add_impl_version}
+
+		EOT
+		exit 0
+	fi
+
+
+	cat <<-EOT
+
+	wasm source files changed but not the spec/impl version and the runtime
+	binary blob. If changes made do not alter logic, just bump 'impl_version'.
+	If they do change logic, bump 'spec_version' and rebuild wasm.
+
+	source file directories:
+	- node/src/runtime
+	- paint
+	- core/sr-*
+
+	versions file: ${VERSIONS_FILE}
+
+	EOT
+
+	# drop through into pushing `gotissues` and exit 1...
+fi
+
+# dropped through. there's something wrong;  exit 1.
+
+exit 1
+
+# vim: noexpandtab
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "*** Initializing WASM build environment"
+
+if [ -z $CI_PROJECT_NAME ] ; then
+   rustup update nightly
+   rustup update stable
+fi
+
+rustup target add wasm32-unknown-unknown --toolchain nightly
@@ -0,0 +1,12 @@
+name: substrate
+version: 0.2
+appVersion: 0.9.1
+description: "Substrate: The platform for blockchain innovators"
+home: https://substrate.network/
+icon: https://substrate.network/favicon.ico
+sources:
+  - https://github.com/paritytech/substrate/
+maintainers:
+  - name: Paritytech Devops Team
+    email: devops-team@parity.io
+tillerVersion: ">=2.8.0"
@@ -0,0 +1,47 @@
+
+
+# Substrate Kubernetes Helm Chart
+
+This [Helm Chart](https://helm.sh/) can be used for deploying containerized 
+**Substrate** to a [Kubernetes](https://kubernetes.io/) cluster.
+
+
+## Prerequisites
+
+- Tested on Kubernetes 1.10.7-gke.6
+
+## Installation
+
+To install the chart with the release name `my-release` into namespace 
+`my-namespace` from within this directory:
+
+```console
+$ helm install --namespace my-namespace --name my-release --values values.yaml ./
+```
+
+The command deploys Substrate on the Kubernetes cluster in the configuration 
+given in `values.yaml`. When the namespace is omitted it'll be installed in 
+the default one.
+
+
+## Removal of the Chart
+
+To uninstall/delete the `my-release` deployment:
+
+```console
+$ helm delete --namespace my-namespace my-release
+```
+
+The command removes all the Kubernetes components associated with the chart and deletes the release.
+
+
+## Upgrading
+
+Once the chart is installed and a new version should be deployed helm takes 
+care of this by
+
+```console
+$ helm upgrade --namespace my-namespace --values values.yaml my-release ./
+```
+
+
@@ -0,0 +1,10 @@
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ .Values.GitlabEnvSlug | default .Values.app }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ .Values.GitlabEnvSlug | default .Values.app }}
+  maxUnavailable: 1
+
@@ -0,0 +1,11 @@
+{{- if .Values.validator.keys }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.app }}-secrets
+  labels:
+    app: {{ .Values.GitlabEnvSlug | default .Values.app }}
+type: Opaque
+data:
+  secrets: {{ .Values.validator.keys | default "" }}
+{{- end }}
@@ -0,0 +1,54 @@
+# see:
+# https://kubernetes.io/docs/tutorials/services/
+# https://kubernetes.io/docs/concepts/services-networking/service/
+# headless service for rpc
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.app }}-rpc
+spec:
+  ports:
+  - port: 9933
+    name: http-rpc
+  - port: 9944
+    name: websocket-rpc
+  selector:
+    app: {{ .Values.GitlabEnvSlug | default .Values.app }}
+  sessionAffinity: None
+  type: ClusterIP
+  clusterIP: None
+---
+{{- if .Values.listen_node_port }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.app }}
+spec:
+  ports:
+  - port: 30333
+    name: p2p
+    nodePort: 30333
+    protocol: TCP
+  selector:
+    app: {{ .Values.GitlabEnvSlug | default .Values.app }}
+  sessionAffinity: None
+  type: NodePort
+  # don't route exteral traffic to non-local pods
+  externalTrafficPolicy: Local
+{{- else if .Values.validator.keys }}
+{{- $root := . -}}
+{{- range until (int .Values.nodes.replicas) }}
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: {{ $root.Values.app }}-{{ . }}
+spec:
+  selector:
+    statefulset.kubernetes.io/pod-name: {{ $root.Values.app }}-{{ . }}
+  ports:
+    - port: 30333
+      targetPort: 30333
+      protocol: TCP
+{{- end }}
+{{- end }}
@@ -0,0 +1,10 @@
+{{- if .Values.rbac.enable }}
+# service account for substrate pods themselves
+# no permissions for the api are required
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: {{ .Values.GitlabEnvSlug | default .Values.app }}
+  name: {{ .Values.rbac.name }}
+{{- end }}
@@ -0,0 +1,139 @@
+# https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/
+# https://cloud.google.com/kubernetes-engine/docs/concepts/statefulset
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ .Values.app }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ .Values.GitlabEnvSlug | default .Values.app }}
+  serviceName: {{ .Values.app }}
+  replicas: {{ .Values.nodes.replicas }}
+  updateStrategy:
+    type: RollingUpdate
+  podManagementPolicy: Parallel
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.GitlabEnvSlug | default .Values.app }}
+    spec:
+      {{- if .Values.rbac.enable }}
+      serviceAccountName: {{ .Values.rbac.name }}
+      {{- else }}
+      serviceAccountName: default
+      {{- end }}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+                - key: node
+                  operator: In
+                  values:
+                  - substrate
+        {{- if .Values.listen_node_port }}
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: "app"
+                    operator: In
+                    values:
+                    - {{ .Values.app }}
+              topologyKey: "kubernetes.io/hostname"
+        {{- end }}
+      terminationGracePeriodSeconds: 300
+      {{- if .Values.validator.keys }}
+      volumes:
+        - name: {{ .Values.app }}-validator-secrets
+          secret:
+            secretName: {{ .Values.app }}-secrets
+      initContainers:
+      - name: prepare-secrets
+        image: busybox
+        command: [ "/bin/sh" ]
+        args:
+          - -c
+          - sed -n -r "s/^${POD_NAME}-key ([^ ]+)$/\1/p" /etc/validator/secrets > {{ .Values.image.basepath }}/key;
+            sed -n -r "s/^${POD_NAME}-node-key ([^ ]+)$/\1/p" /etc/validator/secrets > {{ .Values.image.basepath }}/node-key;
+            sed -n -r "s/^${POD_NAME}-name ([^ ]+)$/\1/p" /etc/validator/secrets > {{ .Values.image.basepath }}/name;
+            test -s {{ .Values.image.basepath }}/name || echo "${POD_NAME}" > {{ .Values.image.basepath }}/name
+        env:
+          # from (workaround for hostname)
+          # https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+        volumeMounts:
+        - name: {{ .Values.app }}-validator-secrets
+          readOnly: true
+          mountPath: "/etc/validator"
+        - name: {{ .Values.app }}dir
+          mountPath: {{ .Values.image.basepath }}
+      {{- end }}
+      containers:
+      - name: {{ .Values.app }}
+        imagePullPolicy: "{{ .Values.image.pullPolicy }}"
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        {{- if .Values.resources }}
+        resources:
+          requests:
+            memory: {{ .Values.resources.memory }}
+            cpu: {{ .Values.resources.cpu }}
+        {{- end }}
+        ports:
+        - containerPort: 30333
+          name: p2p
+        - containerPort: 9933
+          name: http-rpc
+        - containerPort: 9944
+          name: websocket-rpc
+        command: ["/bin/sh"]
+        args:
+          - -c
+          - exec /usr/local/bin/substrate
+            --base-path {{ .Values.image.basepath }}
+            {{- if .Values.validator.keys }}
+            --validator
+            --name $(cat {{ .Values.image.basepath }}/name)
+            --key $(cat {{ .Values.image.basepath }}/key)
+            --node-key $(cat {{ .Values.image.basepath }}/node-key)
+            {{- else }}
+            --name $(POD_NAME)
+            {{- end }}
+            {{- range .Values.nodes.args }} {{ . }} {{- end }}
+        env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+        volumeMounts:
+        - name: {{ .Values.app }}dir
+          mountPath: {{ .Values.image.basepath }}
+        readinessProbe:
+          httpGet:
+            path: /health
+            port: http-rpc
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: http-rpc
+          initialDelaySeconds: 10
+          periodSeconds: 10
+      securityContext:
+        runAsUser: 1000
+        fsGroup: 1000
+  volumeClaimTemplates:
+  - metadata:
+      name: {{ .Values.app }}dir
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      storageClassName: ssd
+      resources:
+        requests:
+          storage: 32Gi
+
@@ -0,0 +1,59 @@
+# set tag manually --set image.tag=latest
+image:
+  repository: parity/substrate
+  tag: latest
+  pullPolicy: Always
+  basepath: /substrate
+
+
+# if set to true a service account for substrate will be created
+rbac:
+  enable: true
+  name: substrate
+
+
+# name of the statefulset
+app: substrate
+listen_node_port: true
+
+nodes:
+  replicas: 2
+  args:
+    # name and data directory are set by the chart itself
+    # key and node-key may be provided on commandline invocation
+    #
+    # - --chain
+    # - krummelanke
+    # serve rpc within the local network
+    # - fenced off the world via firewall
+    # - used for health checks
+    - --rpc-external
+    - --ws-external
+    # - --log
+    # - sub-libp2p=trace
+
+
+validator: {}
+  # providing 'keys' string via --set commandline parameter will run the nodes
+  # in validator mode (--validator).
+  #
+  # name, key and node-key can be given in a base64 encoded keyfile string (at
+  # validator.keys) which has the following format:
+  #
+  # substrate-0-name <node-name>
+  # substrate-0-key <key-seed>
+  # substrate-0-node-key <node-secret-key>
+  # substrate-1-name <node-name>
+  # substrate-1-key <key-seed>
+  # substrate-1-node-key <node-secret-key>
+  #
+  # pod names are canonical. changing these or providing different amount of
+  # keys than the replicas count will lead to behavior noone ever has
+  # experienced before.
+
+
+# maybe adopt resource limits here to the nodes of the pool
+# resources:
+#   memory: "5Gi"
+#   cpu: "1.5"
+
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e
+
+export TERM=xterm
+PROJECT_ROOT=`git rev-parse --show-toplevel`
+
+if [ "$#" -ne 1 ]; then
+  echo "node-template-release.sh path_to_target_archive"
+  exit 1
+fi
+
+PATH_TO_ARCHIVE=$(pwd)/$1
+cd $PROJECT_ROOT/.maintain/node-template-release
+
+cargo run $PROJECT_ROOT/node-template $PATH_TO_ARCHIVE
@@ -0,0 +1,17 @@
+[package]
+name = "node-template-release"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+toml = "0.4"
+tar = "0.4"
+glob = "0.2"
+structopt = "0.3"
+tempfile = "3"
+fs_extra = "1"
+git2 = "0.8"
+flate2 = "1.0"
+
+[workspace]
@@ -0,0 +1,244 @@
+use structopt::StructOpt;
+
+use std::{
+	path::{PathBuf, Path}, collections::HashMap, fs::{File, self}, io::{Read, Write},
+	process::Command
+};
+
+use glob;
+
+use fs_extra::dir::{self, CopyOptions};
+
+use tempfile;
+
+use git2;
+
+use toml;
+
+use tar;
+
+use flate2::{write::GzEncoder, Compression};
+
+const SUBSTRATE_GIT_URL: &str = "https://github.com/paritytech/substrate.git";
+
+type CargoToml = HashMap<String, toml::Value>;
+
+#[derive(StructOpt)]
+struct Options {
+	/// The path to the `node-template` source.
+	#[structopt(parse(from_os_str))]
+	node_template: PathBuf,
+	/// The path where to output the generated `tar.gz` file.
+	#[structopt(parse(from_os_str))]
+	output: PathBuf,
+}
+
+/// Find all `Cargo.toml` files in the given path.
+fn find_cargo_tomls(path: PathBuf) -> Vec<PathBuf> {
+	let path = format!("{}/**/*.toml", path.display());
+
+	let glob = glob::glob(&path).expect("Generates globbing pattern");
+
+	let mut result = Vec::new();
+	glob.into_iter().for_each(|file| {
+		match file {
+			Ok(file) => result.push(file),
+			Err(e) => println!("{:?}", e),
+		}
+	});
+
+	if result.is_empty() {
+		panic!("Did not found any `Cargo.toml` files.");
+	}
+
+	result
+}
+
+/// Copy the `node-template` to the given path.
+fn copy_node_template(node_template: &Path, dest_path: &Path) {
+	let options = CopyOptions::new();
+	dir::copy(node_template, dest_path, &options).expect("Copies node-template to tmp dir");
+}
+
+/// Gets the latest commit id of the repository given by `path`.
+fn get_git_commit_id(path: &Path) -> String {
+	let repo = git2::Repository::discover(path)
+		.expect(&format!("Node template ({}) should be in a git repository.", path.display()));
+
+	let commit_id = repo
+		.head()
+		.expect("Repository should have a head")
+		.peel_to_commit()
+		.expect("Head references a commit")
+		.id();
+
+	format!("{}", commit_id)
+}
+
+/// Parse the given `Cargo.toml` into a `HashMap`
+fn parse_cargo_toml(file: &Path) -> CargoToml {
+	let mut content = String::new();
+	File::open(file).expect("Cargo.toml exists").read_to_string(&mut content).expect("Reads file");
+	toml::from_str(&content).expect("Cargo.toml is a valid toml file")
+}
+
+/// Replaces all substrate path dependencies with a git dependency.
+fn replace_path_dependencies_with_git(cargo_toml_path: &Path, commit_id: &str, cargo_toml: &mut CargoToml) {
+	let mut cargo_toml_path = cargo_toml_path.to_path_buf();
+	// remove `Cargo.toml`
+	cargo_toml_path.pop();
+
+	for table in &["dependencies", "build-dependencies"] {
+		let mut dependencies: toml::value::Table = match cargo_toml
+			.remove(table)
+			.and_then(|v| v.try_into().ok()) {
+			Some(deps) => deps,
+			None => continue,
+		};
+
+		let deps_rewritten = dependencies
+			.iter()
+			.filter_map(|(k, v)| v.clone().try_into::<toml::value::Table>().ok().map(move |v| (k, v)))
+			.filter(|t| t.1.contains_key("path"))
+			.filter(|t| {
+				// if the path does not exists, we need to add this as git dependency
+				t.1.get("path").unwrap().as_str().map(|path| !cargo_toml_path.join(path).exists()).unwrap_or(false)
+			})
+			.map(|(k, mut v)| {
+				// remove `path` and add `git` and `rev`
+				v.remove("path");
+				v.insert("git".into(), SUBSTRATE_GIT_URL.into());
+				v.insert("rev".into(), commit_id.into());
+
+				(k.clone(), v.into())
+			}).collect::<HashMap<_, _>>();
+
+		dependencies.extend(deps_rewritten.into_iter());
+
+		cargo_toml.insert(table.into(), dependencies.into());
+	}
+}
+
+/// Update the top level (workspace) `Cargo.toml` file.
+///
+/// - Adds `profile.release` = `panic = unwind`
+/// - Adds `workspace` definition
+fn update_top_level_cargo_toml(
+	cargo_toml: &mut CargoToml,
+	workspace_members: Vec<&PathBuf>,
+	node_template_path: &Path,
+) {
+	let mut panic_unwind = toml::value::Table::new();
+	panic_unwind.insert("panic".into(), "unwind".into());
+
+	let mut profile = toml::value::Table::new();
+	profile.insert("release".into(), panic_unwind.into());
+
+	cargo_toml.insert("profile".into(), profile.into());
+
+	let members = workspace_members.iter()
+		.map(|p|
+			p.strip_prefix(node_template_path)
+				.expect("Workspace member is a child of the node template path!")
+				.parent()
+				// We get the `Cargo.toml` paths as workspace members, but for the `members` field
+				// we just need the path.
+				.expect("The given path ends with `Cargo.toml` as file name!")
+				.display()
+				.to_string()
+		)
+		.collect::<Vec<_>>();
+
+	let mut members_section = toml::value::Table::new();
+	members_section.insert("members".into(), members.into());
+
+	cargo_toml.insert("workspace".into(), members_section.into());
+}
+
+fn write_cargo_toml(path: &Path, cargo_toml: CargoToml) {
+	let content = toml::to_string_pretty(&cargo_toml).expect("Creates `Cargo.toml`");
+	let mut file = File::create(path).expect(&format!("Creates `{}`.", path.display()));
+	write!(file, "{}", content).expect("Writes `Cargo.toml`");
+}
+
+/// Build and test the generated node-template
+fn build_and_test(path: &Path, cargo_tomls: &[PathBuf]) {
+	// Build node
+	assert!(
+		Command::new("cargo")
+			.args(&["build", "--all"])
+			.current_dir(path)
+			.status()
+			.expect("Compiles node")
+			.success()
+	);
+
+	// Test node
+	assert!(
+		Command::new("cargo")
+			.args(&["test", "--all"])
+			.current_dir(path)
+			.status()
+			.expect("Tests node")
+			.success()
+	);
+
+	// Remove all `target` directories
+	for toml in cargo_tomls {
+		let mut target_path = toml.clone();
+		target_path.pop();
+		target_path = target_path.join("target");
+
+		if target_path.exists() {
+			fs::remove_dir_all(&target_path).expect(&format!("Removes `{}`", target_path.display()));
+		}
+	}
+}
+
+fn main() {
+	let options = Options::from_args();
+
+	let build_dir = tempfile::tempdir().expect("Creates temp build dir");
+
+	let node_template_folder = options
+		.node_template
+		.canonicalize()
+		.expect("Node template path exists")
+		.file_name()
+		.expect("Node template folder is last element of path")
+		.to_owned();
+
+	// The path to the node-template in the build dir.
+	let node_template_path = build_dir.path().join(node_template_folder);
+
+	copy_node_template(&options.node_template, build_dir.path());
+	let cargo_tomls = find_cargo_tomls(build_dir.path().to_owned());
+
+	let commit_id = get_git_commit_id(&options.node_template);
+	let top_level_cargo_toml_path = node_template_path.join("Cargo.toml");
+
+	cargo_tomls.iter().for_each(|t| {
+		let mut cargo_toml = parse_cargo_toml(&t);
+		replace_path_dependencies_with_git(&t, &commit_id, &mut cargo_toml);
+
+		// Check if this is the top level `Cargo.toml`, as this requires some special treatments.
+		if top_level_cargo_toml_path == *t {
+			// All workspace member `Cargo.toml` file paths.
+			let workspace_members = cargo_tomls.iter()
+				.filter(|p| **p != top_level_cargo_toml_path)
+				.collect();
+
+			update_top_level_cargo_toml(&mut cargo_toml, workspace_members, &node_template_path);
+		}
+
+		write_cargo_toml(&t, cargo_toml);
+	});
+
+	build_and_test(&node_template_path, &cargo_tomls);
+
+	let output = GzEncoder::new(File::create(&options.output)
+		.expect("Creates output file"), Compression::default());
+	let mut tar = tar::Builder::new(output);
+	tar.append_dir_all("substrate-node-template", node_template_path)
+		.expect("Writes substrate-node-template archive");
+}
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+# To run this script, you need to install the 'toml' python package and install the 'graphviz' package:
+# pip install toml
+# sudo apt-get install graphviz
+# the first parameter is the runtime folder
+# python ./.maintain/runtime-dep.py ./substrate/runtime | dot -Tpng -o output.png
+import sys
+import os
+import toml
+
+if len(sys.argv) != 2:
+    print("needs the runtime folder.")
+    sys.exit(-1)
+
+runtime_dir = sys.argv[1]
+
+files = [os.path.join(runtime_dir, f, 'Cargo.toml') for f in os.listdir(runtime_dir) if os.path.isfile(os.path.join(runtime_dir, f, 'Cargo.toml')) and f != 'example']
+
+print("digraph G {")
+
+
+PREFIX = "substrate-runtime-"
+
+for f in files:
+    manifest = toml.load(f)
+
+    package_name = manifest['package']['name']
+    deps = [d for d in manifest['dependencies'].keys() if d.startswith(PREFIX)]
+
+    for d in deps:
+        print("    \"{}\" -> \"{}\";".format(package_name, d))
+
+print("}")
@@ -0,0 +1,10 @@
+<script defer type="text/javascript" src="https://substrate.dev/js/config.js"></script>
+<script defer type="text/javascript" src="https://substrate.dev/js/klaro.min.js"></script>
+<script async type="opt-in" data-type="application/javascript"
+		data-src="https://www.googletagmanager.com/gtag/js?id=UA-145158313-2" data-name="googleAnalytics"></script>
+<script type="opt-in" data-type="application/javascript" data-name="googleAnalytics">
+      window.dataLayer = window.dataLayer || [];
+      function gtag(){dataLayer.push(arguments);}
+      gtag('js', new Date());
+      gtag('config', 'UA-145158313-2');
+</script>
@@ -0,0 +1,140 @@
+# Docker compose file to simulate a sentry node setup.
+#
+#
+# Setup:
+#
+# Validator A is not supposed to be connected to the public internet. Instead it
+# connects to a sentry node (sentry-a) which connects to the public internet.
+# Validator B can reach validator A via sentry node A and vice versa.
+#
+#
+# Usage:
+#
+# 1. Build `target/release/substrate` binary: `cargo build --release`
+#
+# 2. Start networks and containers: `sudo docker-compose -f .maintain/sentry-node/docker-compose.yml up`
+#
+# 3. Reach:
+#   - polkadot/apps on localhost:3000
+#   - validator-a: localhost:9944
+#   - validator-b: localhost:9945
+#   - sentry-a: localhost:9946
+
+version: "3.7"
+services:
+
+  validator-a:
+    ports:
+      - "9944:9944"
+    volumes:
+      - ../../target/release/substrate:/usr/local/bin/substrate
+    image: parity/substrate
+    networks:
+      - network-a
+    command:
+      # Local node id: QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR
+      - "--node-key"
+      - "0000000000000000000000000000000000000000000000000000000000000001"
+      - "--base-path"
+      - "/tmp/alice"
+      - "--chain=local"
+      - "--key"
+      - "//Alice"
+      - "--port"
+      - "30333"
+      - "--validator"
+      - "--name"
+      - "AlicesNode"
+      - "--reserved-nodes"
+      - "/dns4/sentry-a/tcp/30333/p2p/QmV7EhW6J6KgmNdr558RH1mPx2xGGznW7At4BhXzntRFsi"
+      # Not only bind to localhost.
+      - "--ws-external"
+      - "--rpc-external"
+      # - "--log"
+      # - "sub-libp2p=trace"
+      # - "--log"
+      # - "afg=trace"
+      - "--no-telemetry"
+      - "--rpc-cors"
+      - "all"
+
+  sentry-a:
+    image: parity/substrate
+    ports:
+      - "9946:9944"
+    volumes:
+      - ../../target/release/substrate:/usr/local/bin/substrate
+    networks:
+      - network-a
+      - internet
+    command:
+      # Local node id: QmV7EhW6J6KgmNdr558RH1mPx2xGGznW7At4BhXzntRFsi
+      - "--node-key"
+      - "0000000000000000000000000000000000000000000000000000000000000003"
+      - "--base-path"
+      - "/tmp/sentry"
+      - "--chain=local"
+      # Don't configure a key, as sentry-a is not a validator.
+      # - "--key"
+      # - "//Charlie"
+      - "--port"
+      - "30333"
+      # sentry-a is not a validator.
+      # - "--validator"
+      - "--name"
+      - "CharliesNode"
+      - "--bootnodes"
+      - "/dns4/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR"
+      - "--bootnodes"
+      - "/dns4/validator-b/tcp/30333/p2p/QmSVnNf9HwVMT1Y4cK1P6aoJcEZjmoTXpjKBmAABLMnZEk"
+      - "--no-telemetry"
+      - "--rpc-cors"
+      - "all"
+      # Not only bind to localhost.
+      - "--ws-external"
+      - "--rpc-external"
+      # Make sure sentry-a still participates as a grandpa voter to forward
+      # grandpa finality gossip messages.
+      - "--grandpa-voter"
+
+  validator-b:
+    image: parity/substrate
+    ports:
+      - "9945:9944"
+    volumes:
+      - ../../target/release/substrate:/usr/local/bin/substrate
+    networks:
+      - internet
+    command:
+      # Local node id: QmSVnNf9HwVMT1Y4cK1P6aoJcEZjmoTXpjKBmAABLMnZEk
+      - "--node-key"
+      - "0000000000000000000000000000000000000000000000000000000000000002"
+      - "--base-path"
+      - "/tmp/bob"
+      - "--chain=local"
+      - "--key"
+      - "//Bob"
+      - "--port"
+      - "30333"
+      - "--validator"
+      - "--name"
+      - "BobsNode"
+      - "--bootnodes"
+      - "/dns4/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR"
+      - "--bootnodes"
+      - "/dns4/sentry-a/tcp/30333/p2p/QmV7EhW6J6KgmNdr558RH1mPx2xGGznW7At4BhXzntRFsi"
+      - "--no-telemetry"
+      - "--rpc-cors"
+      - "all"
+      # Not only bind to localhost.
+      - "--ws-external"
+      - "--rpc-external"
+
+  ui:
+    image: polkadot-js/apps
+    ports:
+      - "3000:80"
+
+networks:
+  network-a:
+  internet:
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+SINGLE_DATES=$(grep -lr "// Copyright [0-9]* Parity Technologies (UK) Ltd.")
+RANGE_DATES=$(grep -lr "// Copyright [0-9]*-[0-9]* Parity Technologies (UK) Ltd.")
+YEAR=$(date +%Y)
+
+for file in $SINGLE_DATES; do
+  FILE_YEAR=$(cat $file | sed -n "s|// Copyright \([[:digit:]][[:digit:]][[:digit:]][[:digit:]]\) Parity Technologies (UK) Ltd.|\1|p")
+  if [ $YEAR -ne $FILE_YEAR ]; then
+    sed -i -e "s|// Copyright \([[:digit:]][[:digit:]][[:digit:]][[:digit:]]\) Parity Technologies (UK) Ltd.|// Copyright \1-$YEAR Parity Technologies (UK) Ltd.|g" $file
+  fi
+done
+
+grep -lr "// Copyright [0-9]*-[0-9]* Parity Technologies (UK) Ltd." |
+  xargs sed -i -e "s|// Copyright \([[:digit:]][[:digit:]][[:digit:]][[:digit:]]\)-[[:digit:]][[:digit:]][[:digit:]][[:digit:]] Parity Technologies (UK) Ltd.|// Copyright \1-$YEAR Parity Technologies (UK) Ltd.|g"
@@ -0,0 +1,9 @@
+#!/bin/sh --
+set -eu
+case $0 in
+   (/*) dir=${0%/*}/;;
+   (*/*) dir=./${0%/*};;
+   (*) dir=.;;
+esac
+
+find "$dir/.." -name Cargo.lock -execdir cargo update \;
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# This script assumes that all pre-requisites are installed.
+
+set -e
+
+PROJECT_ROOT=`git rev-parse --show-toplevel`
+source `dirname "$0"`/common.sh
+
+export CARGO_INCREMENTAL=0
+
+# Save current directory.
+pushd .
+
+cd $ROOT
+
+for SRC in "${SRCS[@]}"
+do
+  echo "*** Updating and building wasm binaries in $SRC"
+  cd "$PROJECT_ROOT/$SRC"
+
+  cargo update
+  ./build.sh "$@"
+
+  cd - >> /dev/null
+done
+
+# Restore initial directory.
+popd