From 1bd19d851107e1ead84a33b237075da4fba56cb4 Mon Sep 17 00:00:00 2001 From: Chevdor Date: Fri, 3 Aug 2018 19:34:53 +0200 Subject: [PATCH] Improve docker image size with a 2 stages image (#463) * Improve docker image size with a 2 stages image * Minor doc updates * Fix and reduce size of the docker image * Fix paths in scripts --- substrate/.dockerignore | 2 ++ substrate/Dockerfile | 38 +++++++++++++++++++++++++++++ substrate/README.adoc | 9 +++---- substrate/ci/script.sh | 5 ++-- substrate/docker/Dockerfile | 33 ------------------------- substrate/docker/build.sh | 18 ++++++++++---- substrate/docker/cleanup.sh | 8 ------ substrate/docker/readme-docker.adoc | 4 +-- substrate/docker/version | 13 ---------- substrate/scripts/build-demos.sh | 28 +++++++++++++++++++++ substrate/{ => scripts}/build.sh | 3 ++- substrate/{ => scripts}/common.sh | 3 +++ substrate/{ => scripts}/init.sh | 2 +- 13 files changed, 96 insertions(+), 70 deletions(-) create mode 100644 substrate/.dockerignore create mode 100644 substrate/Dockerfile delete mode 100644 substrate/docker/Dockerfile delete mode 100755 substrate/docker/cleanup.sh delete mode 100755 substrate/docker/version create mode 100755 substrate/scripts/build-demos.sh rename substrate/{ => scripts}/build.sh (82%) rename substrate/{ => scripts}/common.sh (96%) rename substrate/{ => scripts}/init.sh (86%) diff --git a/substrate/.dockerignore b/substrate/.dockerignore new file mode 100644 index 0000000000..2b0e81eaf0 --- /dev/null +++ b/substrate/.dockerignore @@ -0,0 +1,2 @@ +doc +target diff --git a/substrate/Dockerfile b/substrate/Dockerfile new file mode 100644 index 0000000000..e07e647ce4 --- /dev/null +++ b/substrate/Dockerfile @@ -0,0 +1,38 @@ +FROM phusion/baseimage:0.10.1 as builder +LABEL maintainer "chevdor@gmail.com" +LABEL description="This is the build stage for Polkadot. Here we create the binary." + +ARG PROFILE=release +WORKDIR /polkadot + +COPY . /polkadot + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y cmake pkg-config libssl-dev git + +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \ + export PATH=$PATH:$HOME/.cargo/bin && \ + cargo build --$PROFILE + +# ===== SECOND STAGE ====== + +FROM phusion/baseimage:0.10.0 +LABEL maintainer "chevdor@gmail.com" +LABEL description="This is the 2nd stage: a very small image where we copy the Polkadot binary." +ARG PROFILE=release +COPY --from=builder /polkadot/target/$PROFILE/polkadot /usr/local/bin + +RUN mv /usr/share/ca* /tmp && \ + rm -rf /usr/share/* && \ + mv /tmp/ca-certificates /usr/share/ && \ + rm -rf /usr/lib/python* && \ + mkdir -p /root/.local/share/Polkadot && \ + ln -s /root/.local/share/Polkadot /data + +RUN rm -rf /usr/bin /usr/sbin + +EXPOSE 30333 9933 9944 +VOLUME ["/data"] + +CMD ["/usr/local/bin/polkadot"] diff --git a/substrate/README.adoc b/substrate/README.adoc index f54896e5e2..07a864c55a 100644 --- a/substrate/README.adoc +++ b/substrate/README.adoc @@ -92,7 +92,7 @@ Then build the code: [source, shell] ---- -./build.sh # Builds the WebAssembly binaries +./scripts/build.sh # Builds the WebAssembly binaries cargo build # Builds all native code ---- @@ -118,7 +118,7 @@ The easiest/faster option is to use the latest image. Let´s first check the version we have. The first time you run this command, the polkadot docker image will be downloaded. This takes a bit of time and bandwidth, be patient: [source, shell] -docker run --rm -it chevdor/polkadot:latest ./version +docker run --rm -it chevdor/polkadot:latest pokadot --version .Polkadot arguments @@ -149,8 +149,7 @@ You can either build it yourself (it takes a while...): [source, shell] ---- -cd docker -./build.sh +./docker/build.sh ---- === Reporting issues @@ -159,7 +158,7 @@ If you run into issues with polkadot when using docker, please run the following (replace the tag with the appropriate one if you do not use latest): [source, shell] -docker run --rm -it chevdor/polkadot:latest version +docker run --rm -it chevdor/polkadot:latest polkadot --version This will show you the polkadot version as well as the git commit ref that was used to build your container. Just paste that in the issue you create. diff --git a/substrate/ci/script.sh b/substrate/ci/script.sh index 8d8ded7d7b..dafe2ad5a6 100755 --- a/substrate/ci/script.sh +++ b/substrate/ci/script.sh @@ -22,7 +22,8 @@ case $TARGET in "wasm") # Install prerequisites and build all wasm projects - ./init.sh - ./build.sh + ./scripts/init.sh + ./scripts/build.sh + ./scripts/build-demos.sh ;; esac diff --git a/substrate/docker/Dockerfile b/substrate/docker/Dockerfile deleted file mode 100644 index dfc4ff7787..0000000000 --- a/substrate/docker/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM phusion/baseimage:0.10.1 -LABEL maintainer "chevdor@gmail.com" - -ARG PROFILE=release - -RUN mkdir -p polkadot && \ - apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y cmake pkg-config libssl-dev git && \ - apt-get clean && \ - mkdir -p /root/.local/share/Polkadot && \ - ln -s /root/.local/share/Polkadot /data - -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \ - export PATH=$PATH:$HOME/.cargo/bin && \ - rustup update nightly && \ - rustup target add wasm32-unknown-unknown --toolchain nightly && \ - rustup update stable && \ - cargo install --git https://github.com/alexcrichton/wasm-gc && \ - git clone https://github.com/paritytech/polkadot.git && \ - cd polkadot && \ - ./build.sh && \ - cargo build --$PROFILE && \ - mv target/$PROFILE/polkadot /usr/local/bin && \ - cargo clean && \ - rm -rf /root/.cargo /root/.rustup /tmp/* - -COPY version /polkadot -WORKDIR /polkadot -EXPOSE 30333 9933 9944 -VOLUME ["/data"] - -CMD ["/bin/sh", "polkadot"] diff --git a/substrate/docker/build.sh b/substrate/docker/build.sh index fdbe8c3f81..a4c6831676 100755 --- a/substrate/docker/build.sh +++ b/substrate/docker/build.sh @@ -1,18 +1,26 @@ #!/usr/bin/env bash set -e +pushd . + +# The following line ensure we run from the project root +PROJECT_ROOT=`git rev-parse --show-toplevel` +cd $PROJECT_ROOT + # Find the current version from Cargo.toml -VERSION=`grep "^version" ../Cargo.toml | egrep -o "([0-9\.]+)"` +VERSION=`grep "^version" ./Cargo.toml | egrep -o "([0-9\.]+)"` GITUSER=chevdor GITREPO=polkadot # Build the image -echo "Building ${GITREPO}:$VERSION docker image, hang on!" -time docker build --build-arg PROFILE=release -t ${GITUSER}/${GITREPO}:$VERSION . +echo "Building ${GITUSER}/${GITREPO}:latest docker image, hang on!" +time docker build -f ./docker/Dockerfile --build-arg PROFILE=release -t ${GITUSER}/${GITREPO}:latest . # Show the list of available images for this repo echo "Image is ready" docker images | grep ${GITREPO} -echo -e "\nIf you just built the latest, you may want to update your tag:" -echo " $ docker tag ${GITUSER}/${GITREPO}:$VERSION ${GITUSER}/${GITREPO}:latest" +echo -e "\nIf you just built version ${VERSION}, you may want to update your tag:" +echo " $ docker tag ${GITUSER}/${GITREPO}:$VERSION ${GITUSER}/${GITREPO}:${VERSION}" + +popd diff --git a/substrate/docker/cleanup.sh b/substrate/docker/cleanup.sh deleted file mode 100755 index b4de473a1b..0000000000 --- a/substrate/docker/cleanup.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -# This script helps reduce the size of the built image -# It removes data that is not required. - -export PATH=$PATH:$HOME/.cargo/bin - -cargo clean -rm -rf /root/.cargo /root/.rustup /tmp/* diff --git a/substrate/docker/readme-docker.adoc b/substrate/docker/readme-docker.adoc index bbaceacd7f..6e41f792cc 100644 --- a/substrate/docker/readme-docker.adoc +++ b/substrate/docker/readme-docker.adoc @@ -5,12 +5,12 @@ Run the following command - docker run -d chevdor/polkadot:latest polkadot + docker run -d -P --name polkadot chevdor/polkadot:latest === Building the image To build your own image from the source, you can run the following command: - ./build.sh + ./docker/build.sh NOTE: Building the image takes a while. Count at least 30min on a good machine. diff --git a/substrate/docker/version b/substrate/docker/version deleted file mode 100755 index 047da3302a..0000000000 --- a/substrate/docker/version +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -# This script show the polkadot version and commit ref that was -# used to build the image. -# If you report an issue, call this script to get all details. -# This script will no longer be required once the polkadot cli -# can report its commit ref. - -echo "-----------------------------------------" -printf "Polkadot Docker Container: " -polkadot --version -printf " " -git rev-parse HEAD -echo "-----------------------------------------" diff --git a/substrate/scripts/build-demos.sh b/substrate/scripts/build-demos.sh new file mode 100755 index 0000000000..285da143c1 --- /dev/null +++ b/substrate/scripts/build-demos.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# This script assumes that all pre-requisites are installed. + +set -e + +PROJECT_ROOT=`git rev-parse --show-toplevel` +source `dirname "$0"`/common.sh + +export CARGO_INCREMENTAL=0 + +# Save current directory. +pushd . + +cd $ROOT + +for DEMO in "${DEMOS[@]}" +do + echo "*** Building wasm binaries in $DEMO" + cd "$PROJECT_ROOT/$DEMO" + + ./build.sh + + cd - >> /dev/null +done + +# Restore initial directory. +popd diff --git a/substrate/build.sh b/substrate/scripts/build.sh similarity index 82% rename from substrate/build.sh rename to substrate/scripts/build.sh index f2ced1300a..9cacf74dbb 100755 --- a/substrate/build.sh +++ b/substrate/scripts/build.sh @@ -4,6 +4,7 @@ set -e +PROJECT_ROOT=`git rev-parse --show-toplevel` source `dirname "$0"`/common.sh export CARGO_INCREMENTAL=0 @@ -16,7 +17,7 @@ cd $ROOT for SRC in "${SRCS[@]}" do echo "*** Building wasm binaries in $SRC" - cd $SRC + cd "$PROJECT_ROOT/$SRC" ./build.sh diff --git a/substrate/common.sh b/substrate/scripts/common.sh similarity index 96% rename from substrate/common.sh rename to substrate/scripts/common.sh index 254a4260e4..9d6cc47144 100644 --- a/substrate/common.sh +++ b/substrate/scripts/common.sh @@ -6,6 +6,9 @@ ROOT=`dirname "$0"` SRCS=( "polkadot/runtime/wasm" "substrate/executor/wasm" +) + +DEMOS=( "demo/runtime/wasm" "substrate/test-runtime/wasm" "polkadot/test-parachains/" diff --git a/substrate/init.sh b/substrate/scripts/init.sh similarity index 86% rename from substrate/init.sh rename to substrate/scripts/init.sh index 2bd46709b8..e3618783c1 100755 --- a/substrate/init.sh +++ b/substrate/scripts/init.sh @@ -2,7 +2,7 @@ set -e -echo "*** Initilising WASM build environment" +echo "*** Initialising WASM build environment" rustup update nightly rustup target add wasm32-unknown-unknown --toolchain nightly