From 53f40c583d4b7ce7342e7b2fdefe5a5a351b119c Mon Sep 17 00:00:00 2001 From: gabriel klawitter Date: Fri, 22 Mar 2019 13:13:37 +0100 Subject: [PATCH] Ci update names and chart (#2076) * add option to specifiy substrate node name via environment variable file * gitlab-ci: change job names and adopt to polkadot pipeline * upgrade helm chart Signed-off-by: gabriel --- substrate/.gitlab-ci.yml | 126 +++++++++++------- substrate/scripts/kubernetes/Chart.yaml | 2 +- .../templates/poddisruptionbudget.yaml | 4 +- .../scripts/kubernetes/templates/secrets.yaml | 4 +- .../scripts/kubernetes/templates/service.yaml | 31 ++++- .../kubernetes/templates/serviceaccount.yaml | 2 +- .../kubernetes/templates/statefulset.yaml | 38 +++--- substrate/scripts/kubernetes/values.yaml | 14 +- 8 files changed, 142 insertions(+), 79 deletions(-) diff --git a/substrate/.gitlab-ci.yml b/substrate/.gitlab-ci.yml index b6f5116f7b..f4f42ef49a 100644 --- a/substrate/.gitlab-ci.yml +++ b/substrate/.gitlab-ci.yml @@ -1,5 +1,9 @@ # .gitlab-ci.yml - +# +# substrate +# +# pipelines can be triggered manually in the web +# setting DEPLOY_TAG will only deploy the tagged image stages: @@ -22,7 +26,7 @@ variables: cache: {} -.collect_artifacts: &collect_artifacts +.collect-artifacts: &collect-artifacts artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" when: on_success @@ -32,7 +36,7 @@ cache: {} -.kubernetes_build: &kubernetes_build +.kubernetes-build: &kubernetes-build tags: - kubernetes-parity-build environment: @@ -42,11 +46,11 @@ cache: {} #### stage: merge-test -check:merge:conflict: +check-merge-conflict: stage: merge-test image: parity/tools:latest cache: {} - <<: *kubernetes_build + <<: *kubernetes-build only: - /^[0-9]+$/ variables: @@ -61,11 +65,11 @@ check:merge:conflict: #### stage: test -check:runtime: +check-runtime: stage: test image: parity/tools:latest cache: {} - <<: *kubernetes_build + <<: *kubernetes-build only: - /^[0-9]+$/ variables: @@ -77,10 +81,10 @@ check:runtime: -test:rust:stable: &test +test-linux-stable: &test stage: test cache: - key: "${CI_JOB_NAME}-test" + key: "${CI_JOB_NAME}" paths: - ${CARGO_HOME} - ./target @@ -98,6 +102,9 @@ test:rust:stable: &test - schedules - web - /^[0-9]+$/ + except: + variables: + - $DEPLOY_TAG before_script: - test -d ${CARGO_HOME} -a -d ./target && echo "build cache size:" && @@ -110,7 +117,7 @@ test:rust:stable: &test -.build_only: &build_only +.build-only: &build-only only: - master - tags @@ -119,15 +126,18 @@ test:rust:stable: &test #### stage: build -build:rust:linux:release: &build +build-linux-release: &build stage: build cache: - key: "${CI_JOB_NAME}-build" + key: "${CI_JOB_NAME}" paths: - ${CARGO_HOME} - ./target - <<: *collect_artifacts - <<: *build_only + <<: *collect-artifacts + <<: *build-only + except: + variables: + - $DEPLOY_TAG tags: - linux-docker before_script: @@ -151,7 +161,7 @@ build:rust:linux:release: &build -build:rust:doc:release: &build +build-rust-doc-release: &build stage: build allow_failure: true artifacts: @@ -160,7 +170,7 @@ build:rust:doc:release: &build expire_in: 7 days paths: - ./crate-docs - <<: *build_only + <<: *build-only tags: - linux-docker script: @@ -174,35 +184,38 @@ build:rust:doc:release: &build #### stage: publish -.publish_build: &publish_build +.publish-build: &publish-build stage: publish dependencies: - - build:rust:linux:release + - build-linux-release cache: {} - <<: *build_only - <<: *kubernetes_build + <<: *build-only + <<: *kubernetes-build -publish:docker:release: - <<: *publish_build +publish-docker-release: + <<: *publish-build image: docker:stable services: - docker:dind # collect VERSION artifact here to pass it on to kubernetes - <<: *collect_artifacts + <<: *collect-artifacts variables: DOCKER_HOST: tcp://localhost:2375 DOCKER_DRIVER: overlay2 GIT_STRATEGY: none # DOCKERFILE: scripts/docker/Dockerfile CONTAINER_IMAGE: parity/substrate - script: - - VERSION="$(cat ./artifacts/VERSION)" + before_script: - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" || ( echo "no docker credentials provided"; exit 1 ) - docker login -u "$Docker_Hub_User_Parity" -p "$Docker_Hub_Pass_Parity" - docker info + script: + - VERSION="$(cat ./artifacts/VERSION)" + - echo "Substrate version = ${VERSION}" + - test -z "${VERSION}" && exit 1 - cd ./artifacts - docker build --tag $CONTAINER_IMAGE:$VERSION --tag $CONTAINER_IMAGE:latest . - docker push $CONTAINER_IMAGE:$VERSION @@ -215,8 +228,8 @@ publish:docker:release: -publish:s3:release: - <<: *publish_build +publish-s3-release: + <<: *publish-build image: parity/awscli:latest variables: GIT_STRATEGY: none @@ -237,14 +250,14 @@ publish:s3:release: -publish:s3:doc: +publish-s3-doc: stage: publish allow_failure: true dependencies: - - build:rust:doc:release + - build-rust-doc-release cache: {} - <<: *build_only - <<: *kubernetes_build + <<: *build-only + <<: *kubernetes-build variables: GIT_STRATEGY: none BUCKET: "releases.parity.io" @@ -264,20 +277,15 @@ publish:s3:doc: - - -.deploy:template: &deploy +.deploy-template: &deploy stage: deploy when: manual cache: {} - dependencies: - - publish:docker:release retry: 1 image: parity/kubectl-helm:$HELM_VERSION - <<: *build_only - # variables: - # DEPLOY_TAG: "latest" + <<: *build-only tags: + # this is the runner that is used to deploy it - kubernetes-parity-build before_script: - test -z "${DEPLOY_TAG}" && @@ -292,28 +300,56 @@ publish:s3:doc: --set image.tag=${DEPLOY_TAG} --set validator.keys=${VALIDATOR_KEYS} ./scripts/kubernetes | kubectl apply -f - --dry-run=false - - echo "# substrate namespace" - - kubectl -n substrate get all + - echo "# substrate namespace ${KUBE_NAMESPACE}" + - kubectl -n ${KUBE_NAMESPACE} get all - echo "# substrate's nodes' external ip addresses:" - kubectl get nodes -l node=substrate -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range @.status.addresses[?(@.type=="ExternalIP")]}{.address}{"\n"}{end}' - echo "# substrate' nodes" - - kubectl -n substrate get pods + - kubectl -n ${KUBE_NAMESPACE} get pods -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.nodeName}{"\n"}{end}' + - echo "# wait for the rollout to complete" + - kubectl -n ${KUBE_NAMESPACE} rollout status statefulset/substrate # have environment:url eventually point to the logs -deploy:ew3: + +.deploy-cibuild: &deploy-cibuild <<: *deploy + dependencies: + - publish-docker-release + +.deploy-tag: &deploy-tag + <<: *deploy + only: + variables: + - $DEPLOY_TAG + + + +# have environment:url eventually point to the logs +deploy-ew3: + <<: *deploy-cibuild environment: name: parity-prod-ew3 -deploy:ue1: - <<: *deploy +deploy-ue1: + <<: *deploy-cibuild + environment: + name: parity-prod-ue1 + +deploy-ew3-tag: + <<: *deploy-tag + environment: + name: parity-prod-ew3 + +deploy-ue1-tag: + <<: *deploy-tag environment: name: parity-prod-ue1 + diff --git a/substrate/scripts/kubernetes/Chart.yaml b/substrate/scripts/kubernetes/Chart.yaml index 4dd133c860..8e000ae09f 100644 --- a/substrate/scripts/kubernetes/Chart.yaml +++ b/substrate/scripts/kubernetes/Chart.yaml @@ -1,5 +1,5 @@ name: substrate -version: 0.1 +version: 0.2 appVersion: 0.9.1 description: "Substrate: The platform for blockchain innovators" home: https://substrate.network/ diff --git a/substrate/scripts/kubernetes/templates/poddisruptionbudget.yaml b/substrate/scripts/kubernetes/templates/poddisruptionbudget.yaml index af40522842..56958b1fba 100644 --- a/substrate/scripts/kubernetes/templates/poddisruptionbudget.yaml +++ b/substrate/scripts/kubernetes/templates/poddisruptionbudget.yaml @@ -1,10 +1,10 @@ apiVersion: policy/v1beta1 kind: PodDisruptionBudget metadata: - name: substrate + name: {{ .Values.GitlabEnvSlug | default .Values.app }} spec: selector: matchLabels: - app: substrate + app: {{ .Values.GitlabEnvSlug | default .Values.app }} maxUnavailable: 1 diff --git a/substrate/scripts/kubernetes/templates/secrets.yaml b/substrate/scripts/kubernetes/templates/secrets.yaml index e007384482..97e73ae7ff 100644 --- a/substrate/scripts/kubernetes/templates/secrets.yaml +++ b/substrate/scripts/kubernetes/templates/secrets.yaml @@ -2,9 +2,9 @@ apiVersion: v1 kind: Secret metadata: - name: substrate-secrets + name: {{ .Values.app }}-secrets labels: - app: substrate + app: {{ .Values.GitlabEnvSlug | default .Values.app }} type: Opaque data: secrets: {{ .Values.validator.keys | default "" }} diff --git a/substrate/scripts/kubernetes/templates/service.yaml b/substrate/scripts/kubernetes/templates/service.yaml index 27baa59dc4..eb986adbca 100644 --- a/substrate/scripts/kubernetes/templates/service.yaml +++ b/substrate/scripts/kubernetes/templates/service.yaml @@ -5,9 +5,7 @@ apiVersion: v1 kind: Service metadata: - name: substrate-rpc - labels: - app: substrate + name: {{ .Values.app }}-rpc spec: ports: - port: 9933 @@ -15,15 +13,16 @@ spec: - port: 9944 name: websocket-rpc selector: - app: substrate + app: {{ .Values.GitlabEnvSlug | default .Values.app }} sessionAffinity: None type: ClusterIP clusterIP: None --- +{{- if .Values.listen_node_port }} apiVersion: v1 kind: Service metadata: - name: substrate + name: {{ .Values.app }} spec: ports: - port: 30333 @@ -31,9 +30,27 @@ spec: nodePort: 30333 protocol: TCP selector: - app: substrate + app: {{ .Values.GitlabEnvSlug | default .Values.app }} sessionAffinity: None type: NodePort # don't route exteral traffic to non-local pods externalTrafficPolicy: Local - +{{- else }} +{{- if .Values.validator.keys }} +{{- $root := . -}} +{{- range until (int .Values.nodes.replicas) }} +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ $root.Values.app }}-{{ . }} +spec: + selector: + statefulset.kubernetes.io/pod-name: {{ $root.Values.app }}-{{ . }} + ports: + - port: 30333 + targetPort: 30333 + protocol: TCP +{{- end }} +{{- end }} +{{- end }} diff --git a/substrate/scripts/kubernetes/templates/serviceaccount.yaml b/substrate/scripts/kubernetes/templates/serviceaccount.yaml index 5a0018a121..53d016bffe 100644 --- a/substrate/scripts/kubernetes/templates/serviceaccount.yaml +++ b/substrate/scripts/kubernetes/templates/serviceaccount.yaml @@ -5,6 +5,6 @@ apiVersion: v1 kind: ServiceAccount metadata: labels: - app: substrate + app: {{ .Values.GitlabEnvSlug | default .Values.app }} name: {{ .Values.rbac.name }} {{- end }} diff --git a/substrate/scripts/kubernetes/templates/statefulset.yaml b/substrate/scripts/kubernetes/templates/statefulset.yaml index ec491f6349..27ea85e9b7 100644 --- a/substrate/scripts/kubernetes/templates/statefulset.yaml +++ b/substrate/scripts/kubernetes/templates/statefulset.yaml @@ -3,12 +3,12 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: substrate + name: {{ .Values.app }} spec: selector: matchLabels: - app: substrate - serviceName: substrate + app: {{ .Values.GitlabEnvSlug | default .Values.app }} + serviceName: {{ .Values.app }} replicas: {{ .Values.nodes.replicas }} updateStrategy: type: RollingUpdate @@ -16,7 +16,7 @@ spec: template: metadata: labels: - app: substrate + app: {{ .Values.GitlabEnvSlug | default .Values.app }} spec: {{- if .Values.rbac.enable }} serviceAccountName: {{ .Values.rbac.name }} @@ -32,6 +32,7 @@ spec: operator: In values: - substrate + {{- if .Values.listen_node_port }} podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: @@ -39,14 +40,15 @@ spec: - key: "app" operator: In values: - - substrate + - {{ .Values.app }} topologyKey: "kubernetes.io/hostname" + {{- end }} terminationGracePeriodSeconds: 300 {{- if .Values.validator.keys }} volumes: - - name: substrate-validator-secrets + - name: {{ .Values.app }}-validator-secrets secret: - secretName: substrate-secrets + secretName: {{ .Values.app }}-secrets initContainers: - name: prepare-secrets image: busybox @@ -54,7 +56,9 @@ spec: args: - -c - sed -n -r "s/^${POD_NAME}-key ([^ ]+)$/\1/p" /etc/validator/secrets > {{ .Values.image.basepath }}/key; - sed -n -r "s/^${POD_NAME}-node-key ([^ ]+)$/\1/p" /etc/validator/secrets > {{ .Values.image.basepath }}/node-key + sed -n -r "s/^${POD_NAME}-node-key ([^ ]+)$/\1/p" /etc/validator/secrets > {{ .Values.image.basepath }}/node-key; + sed -n -r "s/^${POD_NAME}-name ([^ ]+)$/\1/p" /etc/validator/secrets > {{ .Values.image.basepath }}/name; + test -z {{ .Values.image.basepath }}/name && echo "${POD_NAME}" > {{ .Values.image.basepath }}/name env: # from (workaround for hostname) # https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/ @@ -63,14 +67,14 @@ spec: fieldRef: fieldPath: metadata.name volumeMounts: - - name: substrate-validator-secrets + - name: {{ .Values.app }}-validator-secrets readOnly: true mountPath: "/etc/validator" - - name: substratedir + - name: {{ .Values.app }}dir mountPath: {{ .Values.image.basepath }} {{- end }} containers: - - name: substrate + - name: {{ .Values.app }} imagePullPolicy: "{{ .Values.image.pullPolicy }}" image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" {{- if .Values.resources }} @@ -91,13 +95,13 @@ spec: - -c - exec /usr/local/bin/substrate --base-path {{ .Values.image.basepath }} - --name $(POD_NAME) - {{- if .Values.validator.enable }} - --validator - {{- end }} {{- if .Values.validator.keys }} + --validator + --name $(cat {{ .Values.image.basepath }}/name) --key $(cat {{ .Values.image.basepath }}/key) --node-key $(cat {{ .Values.image.basepath }}/node-key) + {{- else }} + --name $(POD_NAME) {{- end }} {{- range .Values.nodes.args }} {{ . }} {{- end }} env: @@ -106,7 +110,7 @@ spec: fieldRef: fieldPath: metadata.name volumeMounts: - - name: substratedir + - name: {{ .Values.app }}dir mountPath: {{ .Values.image.basepath }} readinessProbe: httpGet: @@ -125,7 +129,7 @@ spec: fsGroup: 1000 volumeClaimTemplates: - metadata: - name: substratedir + name: {{ .Values.app }}dir spec: accessModes: [ "ReadWriteOnce" ] storageClassName: ssd diff --git a/substrate/scripts/kubernetes/values.yaml b/substrate/scripts/kubernetes/values.yaml index 213ccaebe4..f56cea2993 100644 --- a/substrate/scripts/kubernetes/values.yaml +++ b/substrate/scripts/kubernetes/values.yaml @@ -12,6 +12,10 @@ rbac: name: substrate +# name of the statefulset +app: substrate +listen_node_port: true + nodes: replicas: 2 args: @@ -29,15 +33,17 @@ nodes: # - sub-libp2p=trace -validator: - enable: True - # adds --validator commandline option +validator: {} + # providing 'keys' string via --set commandline parameter will run the nodes + # in validator mode (--validator). # - # key and node-key can be given in a base64 encoded keyfile string (at + # name, key and node-key can be given in a base64 encoded keyfile string (at # validator.keys) which has the following format: # + # substrate-0-name # substrate-0-key # substrate-0-node-key + # substrate-1-name # substrate-1-key # substrate-1-node-key #