From 06c2e023a99341c61e7573443e57f0d96ede7d90 Mon Sep 17 00:00:00 2001 From: Omar Date: Tue, 11 Nov 2025 02:08:36 +0300 Subject: [PATCH] Cleanup Repository & Fix CI (#211) * Move all scripts to a single directory * Switch to cargo-make * Remove the polkadot-sdk from the submodules * WIP: update the CI * Add other jobs to CI * Overhaul the polkadot-sdk caching step * Add a testing step * Fix the CI * Install clang and llvm dependencies * Update the version of clang * Install llvm on macos * Fix ci * Fix ci * Use 1.90.0 version of rust for the polkadot-sdk * Fix CI * Fix CI * Fix CI * Fix CI * Fix CI * Allow warnings * Update runners * Update runners * Simplify CI * Update MacOS runner * Fix zombienet tests * Make cache step faster --- .github/workflows/test.yml | 308 ++++++++---------- .gitignore | 6 +- .gitmodules | 3 - Makefile | 15 - Makefile.toml | 21 ++ .../src/node_implementations/zombienet.rs | 20 +- polkadot-sdk | 1 - run_tests.sh => scripts/run_tests.sh | 0 8 files changed, 173 insertions(+), 201 deletions(-) delete mode 100644 Makefile create mode 100644 Makefile.toml delete mode 160000 polkadot-sdk rename run_tests.sh => scripts/run_tests.sh (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c7a152c..dec54b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,136 +18,95 @@ env: POLKADOT_VERSION: polkadot-stable2506-2 jobs: - cache-polkadot: - name: Build and cache Polkadot binaries on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-24.04, macos-14] - + machete: + name: Check for Unneeded Dependencies + runs-on: ubuntu-24.04 + env: + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" steps: - - name: Checkout repo and submodules + - name: Checkout This Repository uses: actions/checkout@v4 with: submodules: recursive - - - name: Install dependencies (Linux) - if: matrix.os == 'ubuntu-24.04' - run: | - sudo apt-get update - sudo apt-get install -y protobuf-compiler clang libclang-dev - rustup target add wasm32-unknown-unknown - rustup component add rust-src - - - name: Install dependencies (macOS) - if: matrix.os == 'macos-14' - run: | - brew install protobuf - rustup target add wasm32-unknown-unknown - rustup component add rust-src - - - name: Cache binaries - id: cache - uses: actions/cache@v3 + - name: Run Sccache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Install the Rust Toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + - name: Install the Cargo Make Binary + uses: davidB/rust-cargo-make@v1 + - name: Run Cargo Machete + run: cargo make machete + check-fmt: + name: Check Formatting + runs-on: ubuntu-24.04 + env: + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + steps: + - name: Checkout This Repository + uses: actions/checkout@v4 with: - path: | - ~/.cargo/bin/revive-dev-node - ~/.cargo/bin/eth-rpc - key: polkadot-binaries-${{ matrix.os }}-${{ hashFiles('polkadot-sdk/.git') }}-with-dev-node - - - name: Build revive-dev-node - if: steps.cache.outputs.cache-hit != 'true' - run: | - cd polkadot-sdk - cargo install --locked --force --profile=production --path substrate/frame/revive/dev-node/node --bin revive-dev-node - - - name: Build eth-rpc - if: steps.cache.outputs.cache-hit != 'true' - run: | - cd polkadot-sdk - cargo install --path substrate/frame/revive/rpc --bin eth-rpc - - - name: Cache downloaded Polkadot binaries - id: cache-polkadot - uses: actions/cache@v3 + submodules: recursive + - name: Run Sccache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Install the Rust Toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + - name: Install the Cargo Make Binary + uses: davidB/rust-cargo-make@v1 + - name: Run Cargo Formatter + run: cargo make fmt-check + check-clippy: + name: Check Clippy Lints + runs-on: ubuntu-24.04 + env: + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + steps: + - name: Checkout This Repository + uses: actions/checkout@v4 with: - path: | - ~/polkadot-cache/polkadot - ~/polkadot-cache/polkadot-execute-worker - ~/polkadot-cache/polkadot-prepare-worker - ~/polkadot-cache/polkadot-parachain - key: polkadot-downloaded-${{ matrix.os }}-${{ env.POLKADOT_VERSION }} - - - name: Download Polkadot binaries on macOS - if: matrix.os == 'macos-14' && steps.cache-polkadot.outputs.cache-hit != 'true' - run: | - mkdir -p ~/polkadot-cache - curl -sL https://github.com/paritytech/polkadot-sdk/releases/download/${{ env.POLKADOT_VERSION }}/polkadot-aarch64-apple-darwin -o ~/polkadot-cache/polkadot - curl -sL https://github.com/paritytech/polkadot-sdk/releases/download/${{ env.POLKADOT_VERSION }}/polkadot-execute-worker-aarch64-apple-darwin -o ~/polkadot-cache/polkadot-execute-worker - curl -sL https://github.com/paritytech/polkadot-sdk/releases/download/${{ env.POLKADOT_VERSION }}/polkadot-prepare-worker-aarch64-apple-darwin -o ~/polkadot-cache/polkadot-prepare-worker - curl -sL https://github.com/paritytech/polkadot-sdk/releases/download/${{ env.POLKADOT_VERSION }}/polkadot-parachain-aarch64-apple-darwin -o ~/polkadot-cache/polkadot-parachain - chmod +x ~/polkadot-cache/* - - - name: Download Polkadot binaries on Ubuntu - if: matrix.os == 'ubuntu-24.04' && steps.cache-polkadot.outputs.cache-hit != 'true' - run: | - mkdir -p ~/polkadot-cache - curl -sL https://github.com/paritytech/polkadot-sdk/releases/download/${{ env.POLKADOT_VERSION }}/polkadot -o ~/polkadot-cache/polkadot - curl -sL https://github.com/paritytech/polkadot-sdk/releases/download/${{ env.POLKADOT_VERSION }}/polkadot-execute-worker -o ~/polkadot-cache/polkadot-execute-worker - curl -sL https://github.com/paritytech/polkadot-sdk/releases/download/${{ env.POLKADOT_VERSION }}/polkadot-prepare-worker -o ~/polkadot-cache/polkadot-prepare-worker - curl -sL https://github.com/paritytech/polkadot-sdk/releases/download/${{ env.POLKADOT_VERSION }}/polkadot-parachain -o ~/polkadot-cache/polkadot-parachain - chmod +x ~/polkadot-cache/* - - ci: - name: CI on ${{ matrix.os }} + submodules: recursive + - name: Run Sccache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Install the Rust Toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + - name: Install the Cargo Make Binary + uses: davidB/rust-cargo-make@v1 + - name: Run Cargo Clippy + run: cargo make clippy + test: + name: Unit Tests + runs-on: ${{ matrix.os }} needs: cache-polkadot - runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-24.04, macos-14] - + env: + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + POLKADOT_SDK_COMMIT_HASH: "30cda2aad8612a10ff729d494acd9d5353294d63" steps: - - name: Checkout repo + - name: Checkout This Repository uses: actions/checkout@v4 with: submodules: recursive - - - name: Restore binaries from cache - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/revive-dev-node - ~/.cargo/bin/eth-rpc - key: polkadot-binaries-${{ matrix.os }}-${{ hashFiles('polkadot-sdk/.git') }}-with-dev-node - - - name: Restore downloaded Polkadot binaries from cache - uses: actions/cache@v3 - with: - path: | - ~/polkadot-cache/polkadot - ~/polkadot-cache/polkadot-execute-worker - ~/polkadot-cache/polkadot-prepare-worker - ~/polkadot-cache/polkadot-parachain - key: polkadot-downloaded-${{ matrix.os }}-${{ env.POLKADOT_VERSION }} - - - name: Install Polkadot binaries - run: | - sudo cp ~/polkadot-cache/polkadot /usr/local/bin/ - sudo cp ~/polkadot-cache/polkadot-execute-worker /usr/local/bin/ - sudo cp ~/polkadot-cache/polkadot-prepare-worker /usr/local/bin/ - sudo cp ~/polkadot-cache/polkadot-parachain /usr/local/bin/ - sudo chmod +x /usr/local/bin/polkadot* - - - name: Setup Rust toolchain + - name: Run Sccache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Install the Rust Toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: - rustflags: "" - - - name: Add wasm32 target and formatting - run: | - rustup target add wasm32-unknown-unknown - rustup component add rust-src rustfmt clippy - + target: "wasm32-unknown-unknown" + components: "rust-src,rust-std" + - name: Install the Cargo Make Binary + uses: davidB/rust-cargo-make@v1 + - name: Caching Step + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/eth-rpc + ~/.cargo/bin/revive-dev-node + key: polkadot-binaries-${{ env.POLKADOT_SDK_COMMIT_HASH }}-${{ matrix.os }} - name: Install Geth on Ubuntu if: matrix.os == 'ubuntu-24.04' run: | @@ -161,7 +120,7 @@ jobs: # Ubuntu. Eventually, we found out that the last version of geth that worked in our CI was # version 1.15.11. Thus, this is the version that we want to use in CI. The PPA sadly does # not have historic versions of Geth and therefore we need to resort to downloading pre - # built binaries for Geth and the surrounding tools which is what the following parts of + # built binaries for Geth and the surrounding tools which is what the following parts of # the script do. sudo apt-get install -y wget ca-certificates tar @@ -180,7 +139,6 @@ jobs: curl -sL https://github.com/paritytech/revive/releases/download/v0.3.0/resolc-x86_64-unknown-linux-musl -o resolc chmod +x resolc sudo mv resolc /usr/local/bin - - name: Install Geth on macOS if: matrix.os == 'macos-14' run: | @@ -192,69 +150,79 @@ jobs: curl -sL https://github.com/paritytech/revive/releases/download/v0.3.0/resolc-universal-apple-darwin -o resolc chmod +x resolc sudo mv resolc /usr/local/bin - - name: Install Kurtosis on macOS if: matrix.os == 'macos-14' run: brew install kurtosis-tech/tap/kurtosis-cli - - name: Install Kurtosis on Ubuntu if: matrix.os == 'ubuntu-24.04' run: | echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list sudo apt update sudo apt install kurtosis-cli - - - name: Install cargo-machete - uses: clechasseur/rs-cargo@v2 + - name: Run Tests + run: cargo make test + cache-polkadot: + name: Build and Cache Polkadot Binaries on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-24.04, macos-14] + env: + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + RUSTFLAGS: "-Awarnings" + POLKADOT_SDK_COMMIT_HASH: "30cda2aad8612a10ff729d494acd9d5353294d63" + steps: + - name: Caching Step + id: cache-step + uses: actions/cache@v4 with: - command: install - args: cargo-machete@0.7.0 - - name: Machete - run: cargo machete crates + path: | + ~/.cargo/bin/eth-rpc + ~/.cargo/bin/revive-dev-node + key: polkadot-binaries-${{ env.POLKADOT_SDK_COMMIT_HASH }}-${{ matrix.os }} + - name: Checkout the Polkadot SDK Repository + uses: actions/checkout@v4 + if: steps.cache-step.outputs.cache-hit != 'true' + with: + repository: paritytech/polkadot-sdk + ref: ${{ env.POLKADOT_SDK_COMMIT_HASH }} + submodules: recursive + - name: Run Sccache + uses: mozilla-actions/sccache-action@v0.0.9 + if: steps.cache-step.outputs.cache-hit != 'true' + - name: Install the Rust Toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + if: steps.cache-step.outputs.cache-hit != 'true' + with: + target: "wasm32-unknown-unknown" + components: "rust-src" + toolchain: "1.90.0" - - name: Format - run: make format - - - name: Clippy - run: make clippy - - - name: Check revive-dev-node version - run: revive-dev-node --version - - - name: Check eth-rpc version - run: eth-rpc --version - - - name: Check resolc version - run: resolc --version - - - name: Check polkadot version - run: polkadot --version - - - name: Check polkadot-parachain version - run: polkadot-parachain --version - - - name: Check polkadot-execute-worker version - run: polkadot-execute-worker --version - - - name: Check polkadot-prepare-worker version - run: polkadot-prepare-worker --version - - - name: Test Formatting - run: make format - - - name: Test Clippy - run: make clippy - - - name: Test Machete - run: make machete - - - name: Unit Tests - if: matrix.os == 'ubuntu-24.04' - run: cargo test --workspace -- --nocapture - - # We can't install docker in the MacOS image used in CI and therefore we need to skip the - # Kurtosis and lighthouse related tests when running the CI on MacOS. - - name: Unit Tests - if: matrix.os == 'macos-14' + - name: Install dependencies (Linux) + if: matrix.os == 'ubuntu-24.04' && steps.cache-step.outputs.cache-hit != 'true' run: | - cargo test --workspace -- --nocapture --skip lighthouse_geth::tests:: + sudo apt-get update + sudo apt-get install -y protobuf-compiler clang libclang-dev + - name: Install dependencies (macOS) + if: matrix.os == 'macos-14' && steps.cache-step.outputs.cache-hit != 'true' + run: | + brew install protobuf llvm + LLVM_PREFIX="$(brew --prefix llvm)" + echo "LDFLAGS=-L${LLVM_PREFIX}/lib" >> "$GITHUB_ENV" + echo "CPPFLAGS=-I${LLVM_PREFIX}/include" >> "$GITHUB_ENV" + echo "CMAKE_PREFIX_PATH=${LLVM_PREFIX}" >> "$GITHUB_ENV" + echo "LIBCLANG_PATH=${LLVM_PREFIX}/lib" >> "$GITHUB_ENV" + echo "DYLD_FALLBACK_LIBRARY_PATH=${LLVM_PREFIX}/lib" >> "$GITHUB_ENV" + echo "${LLVM_PREFIX}/bin" >> "$GITHUB_PATH" + - name: Build Polkadot Dependencies + if: steps.cache-step.outputs.cache-hit != 'true' + run: | + cargo build \ + --locked \ + --profile production \ + --package revive-dev-node \ + --package pallet-revive-eth-rpc; + mv ./target/production/revive-dev-node ~/.cargo/bin + mv ./target/production/eth-rpc ~/.cargo/bin + chmod +x ~/.cargo/bin/* diff --git a/.gitignore b/.gitignore index 9d46417..1876805 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .DS_Store node_modules /*.json +*.sh # We do not want to commit any log files that we produce from running the code locally so this is # added to the .gitignore file. @@ -13,7 +14,4 @@ workdir !/schema.json !/dev-genesis.json - -# Ignore all shell scripts except for the `run_tests.sh` script -*.sh -!run_tests.sh \ No newline at end of file +!/scripts/* diff --git a/.gitmodules b/.gitmodules index e74b35d..eb732d7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "polkadot-sdk"] - path = polkadot-sdk - url = https://github.com/paritytech/polkadot-sdk.git [submodule "resolc-compiler-tests"] path = resolc-compiler-tests url = https://github.com/paritytech/resolc-compiler-tests diff --git a/Makefile b/Makefile deleted file mode 100644 index 285ae01..0000000 --- a/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -.PHONY: format clippy test machete - -format: - cargo fmt --all -- --check - -clippy: - cargo clippy --all-features --workspace -- --deny warnings - -machete: - cargo install cargo-machete - cargo machete crates - -test: format clippy machete - cargo test --workspace -- --nocapture - diff --git a/Makefile.toml b/Makefile.toml new file mode 100644 index 0000000..5e44f24 --- /dev/null +++ b/Makefile.toml @@ -0,0 +1,21 @@ +[config] +default_to_workspace = false + +[tasks.machete] +command = "cargo" +args = ["machete", "crates"] +install_crate = "cargo-machete" + +[tasks.fmt-check] +command = "cargo" +args = ["fmt", "--all", "--", "--check"] +install_crate = "rustfmt" + +[tasks.clippy] +command = "cargo" +args = ["clippy", "--all-features", "--workspace", "--", "--deny", "warnings"] +install_crate = "clippy" + +[tasks.test] +command = "cargo" +args = ["test", "--workspace", "--", "--nocapture"] diff --git a/crates/node/src/node_implementations/zombienet.rs b/crates/node/src/node_implementations/zombienet.rs index b41645a..aa2be88 100644 --- a/crates/node/src/node_implementations/zombienet.rs +++ b/crates/node/src/node_implementations/zombienet.rs @@ -856,7 +856,7 @@ mod tests { use utils::{new_node, test_config}; #[tokio::test] - #[ignore = "Ignored for the time being"] + #[ignore = "Ignored since CI doesn't have zombienet installed"] async fn test_transfer_transaction_should_return_receipt() { // Arrange let (ctx, node) = new_node().await; @@ -882,6 +882,7 @@ mod tests { } #[test] + #[ignore = "Ignored since CI doesn't have zombienet installed"] fn print_eth_to_polkadot_mappings() { let eth_addresses = vec![ "0x90F8bf6A479f320ead074411a4B0e7944Ea8c9C1", @@ -897,6 +898,7 @@ mod tests { } #[test] + #[ignore = "Ignored since CI doesn't have zombienet installed"] fn test_eth_to_polkadot_address() { let cases = vec![ ( @@ -927,6 +929,7 @@ mod tests { } #[test] + #[ignore = "Ignored since CI doesn't have zombienet installed"] fn eth_rpc_version_works() { // Arrange let context = test_config(); @@ -946,6 +949,7 @@ mod tests { } #[test] + #[ignore = "Ignored since CI doesn't have zombienet installed"] fn version_works() { // Arrange let context = test_config(); @@ -965,7 +969,7 @@ mod tests { } #[tokio::test] - #[ignore = "Ignored since they take a long time to run"] + #[ignore = "Ignored since CI doesn't have zombienet installed"] async fn get_chain_id_from_node_should_succeed() { // Arrange let node = shared_node().await; @@ -984,7 +988,7 @@ mod tests { } #[tokio::test] - #[ignore = "Ignored since they take a long time to run"] + #[ignore = "Ignored since CI doesn't have zombienet installed"] async fn can_get_gas_limit_from_node() { // Arrange let node = shared_node().await; @@ -1002,7 +1006,7 @@ mod tests { } #[tokio::test] - #[ignore = "Ignored since they take a long time to run"] + #[ignore = "Ignored since CI doesn't have zombienet installed"] async fn can_get_coinbase_from_node() { // Arrange let node = shared_node().await; @@ -1020,7 +1024,7 @@ mod tests { } #[tokio::test] - #[ignore = "Ignored since they take a long time to run"] + #[ignore = "Ignored since CI doesn't have zombienet installed"] async fn can_get_block_difficulty_from_node() { // Arrange let node = shared_node().await; @@ -1038,7 +1042,7 @@ mod tests { } #[tokio::test] - #[ignore = "Ignored since they take a long time to run"] + #[ignore = "Ignored since CI doesn't have zombienet installed"] async fn can_get_block_hash_from_node() { // Arrange let node = shared_node().await; @@ -1056,7 +1060,7 @@ mod tests { } #[tokio::test] - #[ignore = "Ignored since they take a long time to run"] + #[ignore = "Ignored since CI doesn't have zombienet installed"] async fn can_get_block_timestamp_from_node() { // Arrange let node = shared_node().await; @@ -1074,7 +1078,7 @@ mod tests { } #[tokio::test] - #[ignore = "Ignored since they take a long time to run"] + #[ignore = "Ignored since CI doesn't have zombienet installed"] async fn can_get_block_number_from_node() { // Arrange let node = shared_node().await; diff --git a/polkadot-sdk b/polkadot-sdk deleted file mode 160000 index 45a0ea7..0000000 --- a/polkadot-sdk +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 45a0ea734fed1ed2b323e4b1bd8f44d53f4db598 diff --git a/run_tests.sh b/scripts/run_tests.sh similarity index 100% rename from run_tests.sh rename to scripts/run_tests.sh