WIP: Subkey docker image (#3183)

* WIP to release subkey im docker

* debug run with publishing

* debug build

* debug build 2

* easier sync

* fix the template

* fix typo

substrate/.gitlab-ci.yml

@@ -106,7 +106,7 @@ cargo-check-subkey:
     - /^v[0-9]+\.[0-9]+.*$/        # i.e. v1.0, v2.1rc1
   script:
     - cd ./subkey
-    - BUILD_DUMMY_WASM_BINARY=1 time cargo check --release   # makes sense to save artifacts for building it
+    - BUILD_DUMMY_WASM_BINARY=1 time cargo check --release
     - sccache -s
 
 
@@ -172,11 +172,13 @@ test-linux-stable-int:
     expire_in:                     24 hrs
     paths:
       - ${CI_COMMIT_SHORT_SHA}_int_failure.log
+  allow_failure:                   true
 
 
 check-web-wasm:
   stage:                           test
   <<:                              *docker-env
+  allow_failure:                   true
   except:
     - /^v[0-9]+\.[0-9]+.*$/        # i.e. v1.0, v2.1rc1
   script:
@@ -218,33 +220,33 @@ build-linux-release:
   stage:                           build
   <<:                              *collect-artifacts
   <<:                              *docker-env
-  <<:                              *build-only
+  # debug <<:                              *build-only
   except:
     variables:
       - $DEPLOY_TAG
   script:
     - time cargo build --release --verbose
-    - mkdir -p ./artifacts
-    - mv ./target/release/substrate ./artifacts/.
+    - mkdir -p ./artifacts/substrate/
+    - mv ./target/release/substrate ./artifacts/substrate/.
     - echo -n "Substrate version = "
     - if [ "${CI_COMMIT_TAG}" ]; then
-        echo "${CI_COMMIT_TAG}" | tee ./artifacts/VERSION;
+        echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION;
       else
-        ./artifacts/substrate --version |
+        ./artifacts/substrate/substrate --version |
         sed -n -r 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' |
-        tee ./artifacts/VERSION;
+        tee ./artifacts/substrate/VERSION;
       fi
-    - sha256sum ./artifacts/substrate | tee ./artifacts/substrate.sha256
+    - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
     - printf '\n# building node-template\n\n'
-    - ./scripts/node-template-release.sh ./artifacts/substrate-node-template.tar.gz
-    - cp -r scripts/docker/* ./artifacts
+    - ./scripts/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
+    - cp -r scripts/docker/substrate.Dockerfile ./artifacts/substrate/
     - sccache -s
 
 build-linux-subkey:
   stage:                           build
   <<:                              *collect-artifacts
   <<:                              *docker-env
-  # <<:                              *build-only
+  # debug <<:                              *build-only
   except:
     variables:
       - $DEPLOY_TAG
@@ -252,19 +254,21 @@ build-linux-subkey:
     - cd ./subkey
     - BUILD_DUMMY_WASM_BINARY=1 time cargo build --release --verbose
     - cd ..
-    # - time cargo build --release
     - sccache -s
-    - mkdir -p ./artifacts
-    - mv ./target/release/subkey ./artifacts/.
+    - mkdir -p ./artifacts/subkey
+    - mv ./target/release/subkey ./artifacts/subkey/.
     - echo -n "Subkey version = "
-    - ./artifacts/subkey --version |
+    - ./artifacts/subkey/subkey --version |
         sed -n -r 's/^subkey ([0-9.]+.*)/\1/p' |
-        tee ./artifacts/SUBKEY-VERSION;
-    - sha256sum ./artifacts/subkey | tee ./artifacts/subkey.sha256
+        tee ./artifacts/subkey/VERSION;
+    - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
+    - cp -r scripts/docker/subkey.Dockerfile ./artifacts/subkey/
+    - sccache -s
 
 build-rust-doc-release:
   stage:                           build
   <<:                              *docker-env
+
   allow_failure:                   true
   artifacts:
     name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
@@ -290,8 +294,13 @@ build-rust-doc-release:
   <<:                              *build-only
   <<:                              *kubernetes-build
 
-publish-docker-release:
-  <<:                              *publish-build
+publish-docker-substrate:          &publish-docker-release
+  # debug <<:                              *publish-build
+  stage:                           publish
+  dependencies:
+    - build-linux-release
+  <<:                              *kubernetes-build
+  # debug
   image:                           docker:stable
   services:
     - docker:dind
@@ -301,7 +310,7 @@ publish-docker-release:
     DOCKER_HOST:                   tcp://localhost:2375
     DOCKER_DRIVER:                 overlay2
     GIT_STRATEGY:                  none
-    # DOCKERFILE:                  scripts/docker/Dockerfile
+    # DOCKERFILE:                  scripts/docker/substrate.Dockerfile
     CONTAINER_IMAGE:               parity/substrate
   before_script:
     - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity"
@@ -309,38 +318,77 @@ publish-docker-release:
     - docker login -u "$Docker_Hub_User_Parity" -p "$Docker_Hub_Pass_Parity"
     - docker info
   script:
-    - VERSION="$(cat ./artifacts/VERSION)"
+    - cd ./artifacts/substrate/
+    - VERSION="$(cat ./VERSION)"
     - echo "Substrate version = ${VERSION}"
     - test -z "${VERSION}" && exit 1
-    - cd ./artifacts
     - docker build
       --build-arg VCS_REF="${CI_COMMIT_SHA}"
       --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
       --tag $CONTAINER_IMAGE:$VERSION
-      --tag $CONTAINER_IMAGE:latest .
+      --tag $CONTAINER_IMAGE:latest
+      --file substrate.Dockerfile .
     - docker push $CONTAINER_IMAGE:$VERSION
     - docker push $CONTAINER_IMAGE:latest
   after_script:
     - docker logout
     # only VERSION information is needed for the deployment
-    - find ./artifacts/ -depth -not -name VERSION -not -name artifacts -delete
+    - find ./artifacts/ -depth -not -name VERSION -type f -delete
+
+publish-docker-subkey:
+  # debug <<:                              *publish-docker-release
+  stage:                           publish
+  dependencies:
+    - build-linux-subkey
+  # debug <<:                              *build-only
+  <<:                              *kubernetes-build
+  image:                           docker:stable
+  services:
+    - docker:dind
+  variables:
+    DOCKER_HOST:                   tcp://localhost:2375
+    DOCKER_DRIVER:                 overlay2
+    GIT_STRATEGY:                  none
+    # DOCKERFILE:                  scripts/docker/subkey.Dockerfile
+    CONTAINER_IMAGE:               parity/subkey
+  before_script:
+    - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity"
+        || ( echo "no docker credentials provided"; exit 1 )
+    - docker login -u "$Docker_Hub_User_Parity" -p "$Docker_Hub_Pass_Parity"
+    - docker info
+  script:
+    - cd ./artifacts/subkey/
+    - VERSION="$(cat ./VERSION)"
+    - echo "Subkey version = ${VERSION}"
+    - test -z "${VERSION}" && exit 1
+    - docker build
+      --build-arg VCS_REF="${CI_COMMIT_SHA}"
+      --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+      --tag $CONTAINER_IMAGE:$VERSION
+      --tag $CONTAINER_IMAGE:latest
+      --file subkey.Dockerfile .
+    - docker push $CONTAINER_IMAGE:$VERSION
+    - docker push $CONTAINER_IMAGE:latest
+  after_script:
+    - docker logout
 
 publish-s3-release:
-  <<:                              *publish-build
+  # debug <<:                              *publish-build
+  stage:                           publish
+  dependencies:
+    - build-linux-release
+    - build-linux-subkey
+  <<:                              *kubernetes-build
+  # debug
   image:                           parity/awscli:latest
   variables:
     GIT_STRATEGY:                  none
     BUCKET:                        "releases.parity.io"
     PREFIX:                        "substrate/${ARCH}-${DOCKER_OS}"
   script:
-    - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/
+    - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/
     - echo "update objects in latest path"
-    - for file in ./artifacts/*; do
-      name="$(basename ${file})";
-      aws s3api copy-object
-        --copy-source ${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/${name}
-        --bucket ${BUCKET} --key ${PREFIX}/latest/${name};
-      done
+    - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
   after_script:
     - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
         --recursive --human-readable --summarize
@@ -423,8 +471,8 @@ publish-gh-doc:
     - kubernetes-parity-build
   before_script:
     - test -z "${DEPLOY_TAG}" &&
-      test -f ./artifacts/VERSION &&
-      DEPLOY_TAG="$(cat ./artifacts/VERSION)"
+        test -f ./artifacts/substrate/VERSION &&
+        DEPLOY_TAG="$(cat ./artifacts/substrate/VERSION)"
     - test "${DEPLOY_TAG}" || ( echo "Neither DEPLOY_TAG nor VERSION information available"; exit 1 )
   script:
     - echo "Substrate version = ${DEPLOY_TAG}"
@@ -450,7 +498,7 @@ publish-gh-doc:
 .deploy-cibuild:                   &deploy-cibuild
   <<:                              *deploy
   dependencies:
-    - publish-docker-release
+    - publish-docker-substrate
 
 .deploy-tag:                       &deploy-tag
   <<:                              *deploy

substrate/scripts/docker/subkey.Dockerfile

@@ -0,0 +1,31 @@
+FROM debian:stretch-slim
+
+# metadata
+ARG VCS_REF
+ARG BUILD_DATE
+
+LABEL io.parity.image.authors="devops-team@parity.io" \
+	io.parity.image.vendor="Parity Technologies" \
+	io.parity.image.title="parity/subkey" \
+	io.parity.image.description="Subkey: key generating utility for Substrate." \
+	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/docker/subkey.Dockerfile" \
+	io.parity.image.revision="${VCS_REF}" \
+	io.parity.image.created="${BUILD_DATE}" \
+	io.parity.image.documentation="https://github.com/paritytech/substrate/tree/${VCS_REF}/subkey"
+
+# show backtraces
+ENV RUST_BACKTRACE 1
+
+# add user
+RUN useradd -m -u 1000 -U -s /bin/sh -d /subkey subkey
+
+# add subkey binary to docker image
+COPY ./subkey /usr/local/bin
+
+USER subkey
+
+# check if executable works in this container
+RUN /usr/local/bin/subkey --version
+
+ENTRYPOINT ["/usr/local/bin/subkey"]
+