Add FFI example (#2037)

* Add FFI example * Remove unnecessary dependency (libc) * Tweak python example and add CI CI Tweak; separate task for ffi-example run * Remove OnceCell dep; use std --------- Co-authored-by: wassimans <wassim@wassimans.com>
2026-06-14 12:11:09 +00:00 · 2025-07-08 12:06:13 +01:00
parent 17b98d0d9e
commit ff6fc1585e
11 changed files with 4241 additions and 1 deletions
@@ -215,13 +215,66 @@ jobs:
      - name: Cargo hack; check each feature/crate on its own
        run: cargo hack --exclude subxt --exclude subxt-signer --exclude subxt-lightclient --exclude subxt-rpcs --exclude-all-features --each-feature check --workspace
-      # Check the parachain-example code, which isn't a part of the workspace so is otherwise ignored.
+      # Check the full examples, which aren't a part of the workspace so are otherwise ignored.
      - name: Cargo check parachain-example
        run: cargo check --manifest-path examples/parachain-example/Cargo.toml
      - name: Cargo check ffi-example
        run: cargo check --manifest-path examples/ffi-example/Cargo.toml
      - if: "failure()"
        uses: "andymckay/cancel-action@a955d435292c0d409d104b57d8e78435a93a6ef1" # v0.5
  ffi_example:
    name: Run FFI Example
    runs-on: ubuntu-latest
    needs: [check]
    steps:
      - name: Checkout sources
        uses: actions/checkout@v4
      - name: Use substrate and polkadot node binaries
        uses: ./.github/workflows/actions/use-nodes
      - name: Install Rust stable toolchain
        uses: actions-rs/toolchain@v1
        with:
          profile: minimal
          toolchain: stable
          target: wasm32-unknown-unknown
          override: true
      - name: Rust Cache
        uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
      - name: Install 
        uses: actions/setup-node@v4
        with:
          # Node version 20 and higher seem to cause an issue with the JS example so stick to 19 for now.
          node-version: 19.x
      - name: Cargo check/run ffi-example
        run: |
          # Start node on port 8000
          substrate-node --dev --rpc-port 8000 > /dev/null 2>&1 &
          # Build the Rust code (hopefully gives long enough for substrate server to start, too):
          cd examples/ffi-example
          cargo build
          # Run the python version of the FFI code:
          echo "Running Python FFI example..."
          python3 src/main.py
          echo "Python FFI example completed with exit code $?"
          # Run the node version of the FFI code
          echo "Installing Node.js dependencies..."
          npm i
          echo "Running Node FFI example..."
          node src/main.js
          echo "Node FFI example completed with exit code $?"
          pkill substrate-node
  wasm_check:
    name: Cargo check (WASM)
    runs-on: ubuntu-latest
@@ -30,6 +30,7 @@ exclude = [
    "signer/wasm-tests",
    "examples/wasm-example",
    "examples/parachain-example",
    "examples/ffi-example",
 ]
 resolver = "2"
@@ -0,0 +1,10 @@
 # MSVC Windows builds of rustc generate these, which store debugging information
 *.pdb
 db.sqlite
 db.sqlite-shm
 db.sqlite-wal
 .tool-versions
 /target
 /build
 /node_modules
@@ -0,0 +1,14 @@
 [package]
 name = "subxt-ffi"
 version = "0.1.0"
 edition = "2024"
 [dependencies]
 hex = "0.4.3"
 subxt = { path = "../../subxt" }
 subxt-signer = { path = "../../signer" }
 tokio = { version = "1", features = ["full"] }
 [lib]
 crate-type = ["cdylib"]
 name = "subxt_ffi"
@@ -0,0 +1,120 @@
 # ffi-example
 This example shows how to expose a small piece of Subxt functionality, in our case, a single balance-transfer call, as a native C-ABI library, consumable from Python and Node.js.
 ## Overview
 - We want to let non-Rust clients interact with any Substrate-based node (Polkadot in this example) via a tiny FFI layer.
 - Instead of exposing Subxt’s full, Rust-centric API, we build a thin **facade crate** that:
  1. Calls Subxt under the hood  
  2. Exposes just the functions we need via `pub extern "C" fn …`  
 - Client languages (Python, JavaScript, Swift, Kotlin, etc.) load the compiled `.so`/`.dylib`/`.dll` and call these C-ABI functions directly.
 ```mermaid
 flowchart LR
  subgraph Rust side
    subxt[Subxt Public API]
    facade[Facade crate]
    node[Substrate node]
    cabi[C ABI library]
    subxt --> facade
    facade --> node
    facade --> cabi
  end
  subgraph Client side
    swift[Swift client]
    python[Python client]
    kotlin[Kotlin client]
    js[JavaScript client]
    swift --> cabi
    python --> cabi
    kotlin --> cabi
    js --> cabi
  end
 ```
 Our one example function is:
 ```rust
 pub extern "C" fn do_transfer(dest_hex: *const c_char, amount: u64) -> i32
 ```
 which does a single balance transfer and returns 0 on success, –1 on error.
 ## Prerequisites
 - Rust toolchain (with cargo)
 - Python 3
 - Node.js (for the JS example. Version 19 worked on my M2 Mac, but version 22 did not, so YMMY).
 - A running Substrate node (Polkadot) on ws://127.0.0.1:8000. One can use Chopsticks for a quick local Polkadot node:
  ```shell
  npx @acala-network/chopsticks \
    --config=https://raw.githubusercontent.com/AcalaNetwork/chopsticks/master/configs/polkadot.yml
  ```
  Or, if you have a `substrate-node` binary, just run `substrate-node --dev  --rpc-port 8000`.
 - In our Python and Javascript files, we introduce a **dest** variable that represents the destination account for the transfer, we gave it a hard coded value (Bob's account public key) from the running Chopsticks node. Feel free to change it to any other account, or better yet, make it generic!
 If you run into any issues running the Node version, I found that I needed to run `brew install python-setuptools` too.
 ## Building
 ### Build the Rust facade library
 ```shell
 cargo build
 ```
 This will produce a dynamic library in target/debug/ (or target/release/ if you pass --release):
 - macOS: libsubxt_ffi.dylib
 - Linux:  libsubxt_ffi.so
 - Windows: subxt_ffi.dll
 ## Running
 ### Python
 #### on macOS / Linux
 ```shell
 python3 src/main.py
 ```
 Expected output:
 ✓ transfer succeeded
 ### Node.js
 #### Install npm dependencies
 In the root of the project run:
 ```shell
 npm install
 ```
 then:
 ``` shell
 node src/main.js
 ```
 Expected output:
 ✓ transfer succeeded
 # Development notes
 - Hex handling: We strip an optional 0x prefix and decode into 32 bytes.
 - FFI safety: We only pass pointers and primitive types (u64, i32) across the boundary.
 - Error codes: We return 0 on success, -1 on any kind of failure (decode error, RPC error, etc.).
 - You can extend this facade crate with any additional functions you need—just expose them as pub extern "C" and follow the same pattern.
 # Limitations
 Translating a complex Rust API like Subxt to a bare bones C ABI ready to be consumed by foreign languages has its limitations. Here's a few of them:
 - Complex types (strings, structs) require to design C-safe representations.
 - Only C primitive types (integers, pointers) are FFI-safe; anything else must be translated.
 - Manual memory management glue code is needed if owned data is returned.
 - Needs a manual translation to every foreign language we export to, every time the Rust library changes.
@@ -0,0 +1,123 @@
 {
  "name": "subxt-ffi",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "subxt-ffi",
      "version": "1.0.0",
      "license": "ISC",
      "dependencies": {
        "ffi-napi": "^4.0.3"
      }
    },
    "node_modules/debug": {
      "version": "4.4.1",
      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz",
      "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==",
      "license": "MIT",
      "dependencies": {
        "ms": "^2.1.3"
      },
      "engines": {
        "node": ">=6.0"
      },
      "peerDependenciesMeta": {
        "supports-color": {
          "optional": true
        }
      }
    },
    "node_modules/ffi-napi": {
      "version": "4.0.3",
      "resolved": "https://registry.npmjs.org/ffi-napi/-/ffi-napi-4.0.3.tgz",
      "integrity": "sha512-PMdLCIvDY9mS32RxZ0XGb95sonPRal8aqRhLbeEtWKZTe2A87qRFG9HjOhvG8EX2UmQw5XNRMIOT+1MYlWmdeg==",
      "hasInstallScript": true,
      "license": "MIT",
      "dependencies": {
        "debug": "^4.1.1",
        "get-uv-event-loop-napi-h": "^1.0.5",
        "node-addon-api": "^3.0.0",
        "node-gyp-build": "^4.2.1",
        "ref-napi": "^2.0.1 || ^3.0.2",
        "ref-struct-di": "^1.1.0"
      },
      "engines": {
        "node": ">=10"
      }
    },
    "node_modules/get-symbol-from-current-process-h": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/get-symbol-from-current-process-h/-/get-symbol-from-current-process-h-1.0.2.tgz",
      "integrity": "sha512-syloC6fsCt62ELLrr1VKBM1ggOpMdetX9hTrdW77UQdcApPHLmf7CI7OKcN1c9kYuNxKcDe4iJ4FY9sX3aw2xw==",
      "license": "MIT"
    },
    "node_modules/get-uv-event-loop-napi-h": {
      "version": "1.0.6",
      "resolved": "https://registry.npmjs.org/get-uv-event-loop-napi-h/-/get-uv-event-loop-napi-h-1.0.6.tgz",
      "integrity": "sha512-t5c9VNR84nRoF+eLiz6wFrEp1SE2Acg0wS+Ysa2zF0eROes+LzOfuTaVHxGy8AbS8rq7FHEJzjnCZo1BupwdJg==",
      "license": "MIT",
      "dependencies": {
        "get-symbol-from-current-process-h": "^1.0.1"
      }
    },
    "node_modules/ms": {
      "version": "2.1.3",
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
      "license": "MIT"
    },
    "node_modules/node-addon-api": {
      "version": "3.2.1",
      "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz",
      "integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==",
      "license": "MIT"
    },
    "node_modules/node-gyp-build": {
      "version": "4.8.4",
      "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz",
      "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==",
      "license": "MIT",
      "bin": {
        "node-gyp-build": "bin.js",
        "node-gyp-build-optional": "optional.js",
        "node-gyp-build-test": "build-test.js"
      }
    },
    "node_modules/ref-napi": {
      "version": "3.0.3",
      "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-3.0.3.tgz",
      "integrity": "sha512-LiMq/XDGcgodTYOMppikEtJelWsKQERbLQsYm0IOOnzhwE9xYZC7x8txNnFC9wJNOkPferQI4vD4ZkC0mDyrOA==",
      "hasInstallScript": true,
      "license": "MIT",
      "dependencies": {
        "debug": "^4.1.1",
        "get-symbol-from-current-process-h": "^1.0.2",
        "node-addon-api": "^3.0.0",
        "node-gyp-build": "^4.2.1"
      },
      "engines": {
        "node": ">= 10.0"
      }
    },
    "node_modules/ref-struct-di": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/ref-struct-di/-/ref-struct-di-1.1.1.tgz",
      "integrity": "sha512-2Xyn/0Qgz89VT+++WP0sTosdm9oeowLP23wRJYhG4BFdMUrLj3jhwHZNEytYNYgtPKLNTP3KJX4HEgBvM1/Y2g==",
      "license": "MIT",
      "dependencies": {
        "debug": "^3.1.0"
      }
    },
    "node_modules/ref-struct-di/node_modules/debug": {
      "version": "3.2.7",
      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
      "license": "MIT",
      "dependencies": {
        "ms": "^2.1.1"
      }
    }
  }
 }
@@ -0,0 +1,23 @@
 {
  "name": "subxt-ffi",
  "version": "1.0.0",
  "description": "Example of exposing some Subxt functionality to other languages through FFI",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "repository": {
    "type": "git",
    "url": "git+https://github.com/wassimans/subxt-ffi.git"
  },
  "keywords": [],
  "author": "",
  "license": "ISC",
  "bugs": {
    "url": "https://github.com/wassimans/subxt-ffi/issues"
  },
  "homepage": "https://github.com/wassimans/subxt-ffi#readme",
  "dependencies": {
    "ffi-napi": "^4.0.3"
  }
 }
@@ -0,0 +1,73 @@
 use hex::decode;
 use std::{ffi::CStr, os::raw::c_char, sync::OnceLock};
 use subxt::{OnlineClient, PolkadotConfig, dynamic::Value, ext::scale_value::Composite, tx};
 use subxt_signer::sr25519::dev;
 use tokio::runtime::Runtime;
 static TOKIO: OnceLock<Runtime> = OnceLock::new();
 fn tokio_rt() -> &'static Runtime {
    TOKIO.get_or_init(|| Runtime::new().expect("failed to start tokio"))
 }
 /// A simple C‐ABI function to transfer `amount` to a hex‐encoded `dest`.
 /// Assumes a running node’s WS endpoint is at ws://127.0.0.1:8000
 #[unsafe(no_mangle)]
 pub extern "C" fn do_transfer(dest_hex: *const c_char, amount: u64) -> i32 {
    let amount = amount as u128;
    // We need to convert C string to Rust str
    let raw_s = unsafe { CStr::from_ptr(dest_hex).to_str().unwrap_or_default() };
    // Strip optional 0x prefix
    let s = raw_s.strip_prefix("0x").unwrap_or(raw_s);
    // Decode hex, force a 32‐byte AccountId
    let raw = decode(s).expect("hex decode");
    let arr: [u8; 32] = raw.as_slice().try_into().expect("must be 32 bytes");
    // Wrap into a MultiAddress::Id variant for dynamic calls:
    let dst = Value::variant(
        "Id",
        Composite::unnamed(vec![
            // scale encode
            Value::from_bytes(arr.to_vec()),
        ]),
    );
    // Spin up (or reuse) our Tokio runtime and connect:
    let client = tokio_rt().block_on(async {
        OnlineClient::<PolkadotConfig>::from_url("ws://127.0.0.1:8000")
            .await
            .unwrap()
    });
    let signer = dev::alice();
    // Build the dynamic metadata extrinsic:
    let tx = tx::dynamic(
        "Balances",
        "transfer_keep_alive",
        vec![
            dst.clone(),
            // primitive numeric value
            Value::u128(amount),
        ],
    );
    // Submit and wait for finalize
    let res: Result<_, subxt::Error> = tokio_rt().block_on(async {
        let progress = client
            .tx()
            .sign_and_submit_then_watch_default(&tx, &signer)
            .await?;
        progress.wait_for_finalized_success().await
    });
    // Return code
    match res {
        Ok(_) => 0,
        Err(e) => {
            // print the Subxt error
            eprintln!("do_transfer failed: {:#?}", e);
            -1
        }
    }
 }
@@ -0,0 +1,28 @@
 const path = require("path");
 const ffi = require("ffi-napi");
 // Pick the correct library file name depending on the platform
 const libPath = path.resolve(__dirname, "../target/debug", {
  darwin: "libsubxt_ffi.dylib",
  linux:  "libsubxt_ffi.so",
  win32:  "subxt_ffi.dll"
 }[process.platform]);
 // Declare the FFI interface
 const lib = ffi.Library(libPath, {
  "do_transfer": ["int", ["string", "uint64"]]
 });
 function doTransfer(destHex, amount) {
  const code = lib.do_transfer(destHex, amount);
  if (code === 0) {
    console.log("✓ transfer succeeded");
  } else {
    console.error("✗ transfer failed, code =", code);
  }
 }
 // Example usage:
 const dest    = "0x8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48";
 const amount  = 1_000_000_000_000;            // fits in u64
 doTransfer(dest, amount);
@@ -0,0 +1,42 @@
 #!/usr/bin/env python3
 import os, sys, ctypes, platform
 from ctypes import c_char_p, c_uint64, c_int
 # The library name depends on the playform type
 if platform.system() == "Linux":
    libname = "libsubxt_ffi.so"
 elif platform.system() == "Darwin":
    libname = "libsubxt_ffi.dylib"
 elif platform.system() == "Windows":
    libname = "subxt_ffi.dll"
 else:
    raise RuntimeError(f"Unsupported platform: {platform.system()}")
 # Load the library
 lib_path = os.path.join(os.path.dirname(__file__), "..", "target", "debug", libname)
 lib = ctypes.CDLL(lib_path)
 # Tell ctypes about our function signature, the one we defined in the Rust library
 lib.do_transfer.argtypes = (c_char_p, c_uint64)
 lib.do_transfer.restype  = c_int
 def do_transfer(dest_hex: str, amount: int) -> int:
    """
    Perform a transfer.
    dest_hex:  hex-string of the 32-byte AccountId (e.g. "0x...")
    amount:    integer amount (fits in u64)
    Returns 0 on success, –1 on error.
    """
    # ensure we pass a C-string pointer
    dest_bytes = dest_hex.encode("utf8")
    return lib.do_transfer(dest_bytes, amount)
 if __name__ == "__main__":
    # example usage
    dest = "0x8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48"
    amt  = 1_000_000_000_000
    code = do_transfer(dest, amt)
    if code == 0:
        print("✓ transfer succeeded")
    else:
        print("✗ transfer failed")