From ea8cb96ceaf5aeb19e834fde9a4a59dea73b556d Mon Sep 17 00:00:00 2001 From: Satoshi Qazi Muhammed Date: Mon, 19 Oct 2026 21:08:01 +0400 Subject: [PATCH] chore(papi): commit the sources behind the four polkadot-api packages papi-utils, bizinikiwi-bindings, metadata-builders and merkleize-metadata held only dist/ builds, and no repository held their source. Every source map in dist/ embeds the TypeScript it was compiled from; that is now committed as src/ (59 files), and it matches upstream polkadot-api (MIT) exactly once @polkadot-api/* imports are renamed: papi-utils @polkadot-api/utils 0.2.0 bizinikiwi-bindings @polkadot-api/substrate-bindings 0.16.6 metadata-builders @polkadot-api/metadata-builders 0.13.0-0.13.2 merkleize-metadata @polkadot-api/merkleize-metadata 1.1.21-1.1.23 scripts/check-papi-sources.mjs, run in the lint job, fails unless the source embedded in dist/ is exactly the committed src/. tsconfig.json excludes these src/ trees; ESLint already skipped the packages. Each README now states the upstream package and version. --- .github/workflows/pr-any.yml | 5 + eslint.config.js | 3 +- packages/bizinikiwi-bindings/README.md | 13 +- .../src/codecs/blockHeader.ts | 47 ++ .../src/codecs/metadata/deprecation.ts | 26 + .../src/codecs/metadata/docs.ts | 3 + .../src/codecs/metadata/lookup.ts | 89 +++ .../src/codecs/metadata/metadata.ts | 78 +++ .../src/codecs/metadata/pallets.ts | 105 ++++ .../src/codecs/metadata/runtime-api.ts | 38 ++ .../src/codecs/metadata/unified.ts | 161 ++++++ .../src/codecs/metadata/v14.ts | 33 ++ .../src/codecs/metadata/v15.ts | 35 ++ .../src/codecs/metadata/v16.ts | 34 ++ .../src/codecs/scale/AccountId.ts | 25 + .../src/codecs/scale/Binary.ts | 106 ++++ .../src/codecs/scale/BitSeq.ts | 53 ++ .../src/codecs/scale/Hex.ts | 20 + .../src/codecs/scale/Self.ts | 28 + .../src/codecs/scale/Variant.ts | 149 +++++ .../src/codecs/scale/bitSequence.ts | 30 + .../src/codecs/scale/char.ts | 7 + .../src/codecs/scale/compact.ts | 4 + .../src/codecs/scale/ethAccount.ts | 39 ++ .../src/codecs/scale/fixed-str.ts | 11 + .../src/codecs/scale/shaped.ts | 100 ++++ .../src/codecs/scale/with-inner.ts | 5 + .../src/extrinsics/extrinsic-format.ts | 28 + .../bizinikiwi-bindings/src/hashes/blake2.ts | 11 + .../bizinikiwi-bindings/src/hashes/blake3.ts | 8 + .../bizinikiwi-bindings/src/hashes/h64.ts | 177 ++++++ .../src/hashes/identity.ts | 1 + .../bizinikiwi-bindings/src/hashes/keccak.ts | 3 + .../bizinikiwi-bindings/src/hashes/twoX.ts | 28 + packages/bizinikiwi-bindings/src/storage.ts | 106 ++++ .../src/trie/node-decoder.ts | 70 +++ .../bizinikiwi-bindings/src/trie/proofs.ts | 72 +++ .../bizinikiwi-bindings/src/trie/types.ts | 55 ++ .../bizinikiwi-bindings/src/types/enum.ts | 93 +++ .../bizinikiwi-bindings/src/utils/multisig.ts | 34 ++ .../src/utils/ss58-util.ts | 93 +++ packages/merkleize-metadata/README.md | 13 +- packages/merkleize-metadata/src/codecs.ts | 152 +++++ .../src/decode-and-collect.ts | 135 +++++ .../src/get-accessible-types.ts | 60 ++ packages/merkleize-metadata/src/get-lookup.ts | 119 ++++ .../merkleize-metadata/src/get-metadata.ts | 18 + packages/merkleize-metadata/src/main.ts | 294 ++++++++++ packages/merkleize-metadata/src/proof.ts | 57 ++ packages/merkleize-metadata/src/utils.ts | 15 + packages/metadata-builders/README.md | 13 +- .../metadata-builders/src/checksum-builder.ts | 530 ++++++++++++++++++ .../metadata-builders/src/dynamic-builder.ts | 191 +++++++ .../src/lookup-codec-builder.ts | 119 ++++ .../metadata-builders/src/lookup-graph.ts | 214 +++++++ packages/metadata-builders/src/lookups.ts | 456 +++++++++++++++ packages/metadata-builders/src/with-cache.ts | 44 ++ packages/papi-utils/README.md | 13 +- packages/papi-utils/src/AbortError.ts | 6 + packages/papi-utils/src/filterObject.ts | 8 + packages/papi-utils/src/hex.ts | 57 ++ packages/papi-utils/src/jsonPrint.ts | 11 + packages/papi-utils/src/mapObject.ts | 29 + packages/papi-utils/src/mergeUint8.ts | 25 + packages/papi-utils/src/noop.ts | 1 + scripts/check-papi-sources.mjs | 71 +++ tsconfig.json | 7 +- 67 files changed, 4678 insertions(+), 6 deletions(-) create mode 100644 packages/bizinikiwi-bindings/src/codecs/blockHeader.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/metadata/deprecation.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/metadata/docs.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/metadata/lookup.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/metadata/metadata.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/metadata/pallets.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/metadata/runtime-api.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/metadata/unified.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/metadata/v14.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/metadata/v15.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/metadata/v16.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/scale/AccountId.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/scale/Binary.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/scale/BitSeq.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/scale/Hex.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/scale/Self.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/scale/Variant.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/scale/bitSequence.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/scale/char.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/scale/compact.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/scale/ethAccount.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/scale/fixed-str.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/scale/shaped.ts create mode 100644 packages/bizinikiwi-bindings/src/codecs/scale/with-inner.ts create mode 100644 packages/bizinikiwi-bindings/src/extrinsics/extrinsic-format.ts create mode 100644 packages/bizinikiwi-bindings/src/hashes/blake2.ts create mode 100644 packages/bizinikiwi-bindings/src/hashes/blake3.ts create mode 100644 packages/bizinikiwi-bindings/src/hashes/h64.ts create mode 100644 packages/bizinikiwi-bindings/src/hashes/identity.ts create mode 100644 packages/bizinikiwi-bindings/src/hashes/keccak.ts create mode 100644 packages/bizinikiwi-bindings/src/hashes/twoX.ts create mode 100644 packages/bizinikiwi-bindings/src/storage.ts create mode 100644 packages/bizinikiwi-bindings/src/trie/node-decoder.ts create mode 100644 packages/bizinikiwi-bindings/src/trie/proofs.ts create mode 100644 packages/bizinikiwi-bindings/src/trie/types.ts create mode 100644 packages/bizinikiwi-bindings/src/types/enum.ts create mode 100644 packages/bizinikiwi-bindings/src/utils/multisig.ts create mode 100644 packages/bizinikiwi-bindings/src/utils/ss58-util.ts create mode 100644 packages/merkleize-metadata/src/codecs.ts create mode 100644 packages/merkleize-metadata/src/decode-and-collect.ts create mode 100644 packages/merkleize-metadata/src/get-accessible-types.ts create mode 100644 packages/merkleize-metadata/src/get-lookup.ts create mode 100644 packages/merkleize-metadata/src/get-metadata.ts create mode 100644 packages/merkleize-metadata/src/main.ts create mode 100644 packages/merkleize-metadata/src/proof.ts create mode 100644 packages/merkleize-metadata/src/utils.ts create mode 100644 packages/metadata-builders/src/checksum-builder.ts create mode 100644 packages/metadata-builders/src/dynamic-builder.ts create mode 100644 packages/metadata-builders/src/lookup-codec-builder.ts create mode 100644 packages/metadata-builders/src/lookup-graph.ts create mode 100644 packages/metadata-builders/src/lookups.ts create mode 100644 packages/metadata-builders/src/with-cache.ts create mode 100644 packages/papi-utils/src/AbortError.ts create mode 100644 packages/papi-utils/src/filterObject.ts create mode 100644 packages/papi-utils/src/hex.ts create mode 100644 packages/papi-utils/src/jsonPrint.ts create mode 100644 packages/papi-utils/src/mapObject.ts create mode 100644 packages/papi-utils/src/mergeUint8.ts create mode 100644 packages/papi-utils/src/noop.ts create mode 100644 scripts/check-papi-sources.mjs diff --git a/.github/workflows/pr-any.yml b/.github/workflows/pr-any.yml index 9d0d4f9..541eb81 100644 --- a/.github/workflows/pr-any.yml +++ b/.github/workflows/pr-any.yml @@ -31,3 +31,8 @@ jobs: - name: check built manifests if: matrix.step == 'build' run: python3 scripts/check-built-manifests.py . + # The four polkadot-api packages ship committed dist/ builds; their source + # maps must embed exactly the src/ committed beside them. + - name: check papi sources + if: matrix.step == 'lint' + run: node scripts/check-papi-sources.mjs diff --git a/eslint.config.js b/eslint.config.js index ef9dcff..c184822 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -9,7 +9,8 @@ export default [ // see the tsconfig.eslint.json for explanation 'packages/api-augment/src/dicle/*.ts', 'packages/api-augment/src/pezkuwi/*.ts', - // compiled packages without sources here; there is nothing to lint + // upstream polkadot-api packages (src/ + committed dist/), kept in upstream's style; + // scripts/check-papi-sources.mjs ties dist/ to src/ 'packages/bizinikiwi-bindings/**', 'packages/merkleize-metadata/**', 'packages/metadata-builders/**', diff --git a/packages/bizinikiwi-bindings/README.md b/packages/bizinikiwi-bindings/README.md index 6272526..3eb7b6a 100644 --- a/packages/bizinikiwi-bindings/README.md +++ b/packages/bizinikiwi-bindings/README.md @@ -1 +1,12 @@ -# @polkadot-api/substrate-bindings +# @pezkuwi/bizinikiwi-bindings + +**Source:** `src/` is [`@polkadot-api/substrate-bindings`](https://www.npmjs.com/package/@polkadot-api/substrate-bindings/v/0.16.6) 0.16.6 from +[polkadot-api](https://github.com/polkadot-api/polkadot-api) (MIT), with imports of +`@polkadot-api/*` renamed to `@pezkuwi/*` (`substrate-bindings` → `bizinikiwi-bindings`). `dist/` is its build; +`scripts/check-papi-sources.mjs` fails CI unless the source embedded in `dist/` equals `src/`. + +The upstream README follows. + +--- + +## @polkadot-api/substrate-bindings diff --git a/packages/bizinikiwi-bindings/src/codecs/blockHeader.ts b/packages/bizinikiwi-bindings/src/codecs/blockHeader.ts new file mode 100644 index 0000000..ba88ead --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/blockHeader.ts @@ -0,0 +1,47 @@ +import { + Bytes, + CodecType, + Struct, + Vector, + _void, + enhanceCodec, + Hex, + Variant, + compactNumber, +} from "./scale" + +const textEncoder = new TextEncoder() +const textDecoder = new TextDecoder() + +const fourChars = enhanceCodec( + Bytes(4), + textEncoder.encode.bind(textEncoder), + textDecoder.decode.bind(textDecoder), +) + +const diggestVal = Struct({ + engine: fourChars, + payload: Hex(), +}) + +const diggest = Variant( + { + other: Bytes(), + consensus: diggestVal, + seal: diggestVal, + preRuntime: diggestVal, + runtimeUpdated: _void, + }, + [0, 4, 5, 6, 8], +) + +const hex32 = Hex(32) +export const blockHeader = Struct({ + parentHash: hex32, + number: compactNumber, + stateRoot: hex32, + extrinsicRoot: hex32, + digests: Vector(diggest), +}) + +export type BlockHeader = CodecType diff --git a/packages/bizinikiwi-bindings/src/codecs/metadata/deprecation.ts b/packages/bizinikiwi-bindings/src/codecs/metadata/deprecation.ts new file mode 100644 index 0000000..03683c5 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/metadata/deprecation.ts @@ -0,0 +1,26 @@ +import { _void, Enum, Option, str, Struct, u8, Vector } from "scale-ts" + +export const itemDeprecation = Enum({ + NotDeprecated: _void, + DeprecatedWithoutNote: _void, + Deprecated: Struct({ + note: str, + since: Option(str), + }), +}) + +export const variantDeprecation = Vector( + Struct({ + index: u8, + deprecation: Enum( + { + DeprecatedWithoutNote: _void, + Deprecated: Struct({ + note: str, + since: Option(str), + }), + }, + [1, 2], + ), + }), +) diff --git a/packages/bizinikiwi-bindings/src/codecs/metadata/docs.ts b/packages/bizinikiwi-bindings/src/codecs/metadata/docs.ts new file mode 100644 index 0000000..1538f01 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/metadata/docs.ts @@ -0,0 +1,3 @@ +import { Vector, str } from "scale-ts" + +export const docs = Vector(str) diff --git a/packages/bizinikiwi-bindings/src/codecs/metadata/lookup.ts b/packages/bizinikiwi-bindings/src/codecs/metadata/lookup.ts new file mode 100644 index 0000000..407a4cc --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/metadata/lookup.ts @@ -0,0 +1,89 @@ +import { compactNumber } from "../scale/compact" +import { + CodecType, + Enum, + Option, + Struct, + Vector, + _void, + str, + u32, + u8, +} from "scale-ts" +import { docs } from "./docs" + +const oStr = Option(str) + +const primitive = Enum({ + bool: _void, + char: _void, + str: _void, + u8: _void, + u16: _void, + u32: _void, + u64: _void, + u128: _void, + u256: _void, + i8: _void, + i16: _void, + i32: _void, + i64: _void, + i128: _void, + i256: _void, +}) + +const fields = Vector( + Struct({ + name: oStr, + type: compactNumber, + typeName: oStr, + docs, + }), +) + +const arr = Struct({ + len: u32, + type: compactNumber, +}) + +const bitSequence = Struct({ + bitStoreType: compactNumber, + bitOrderType: compactNumber, +}) + +const variant = Vector( + Struct({ + name: str, + fields, + index: u8, + docs, + }), +) + +const def = Enum({ + composite: fields, + variant, + sequence: compactNumber, + array: arr, + tuple: Vector(compactNumber), + primitive, + compact: compactNumber, + bitSequence, +}) + +const param = Struct({ + name: str, + type: Option(compactNumber), +}) +const params = Vector(param) + +const entry = Struct({ + id: compactNumber, + path: docs, + params, + def, + docs, +}) + +export const lookup = Vector(entry) +export type V14Lookup = CodecType diff --git a/packages/bizinikiwi-bindings/src/codecs/metadata/metadata.ts b/packages/bizinikiwi-bindings/src/codecs/metadata/metadata.ts new file mode 100644 index 0000000..70646d8 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/metadata/metadata.ts @@ -0,0 +1,78 @@ +import { + Bytes, + type Codec, + type CodecType, + Enum, + Option, + Struct, + Tuple, + compact, + createCodec, + u32, +} from "scale-ts" +import type { HexString } from "../scale" +import { v14 } from "./v14" +import { v15 } from "./v15" +import { v16 } from "./v16" + +const unsupportedFn = () => { + throw new Error("Unsupported metadata version!") +} + +const unsupported = createCodec( + unsupportedFn, + unsupportedFn, +) as unknown as Codec + +export const metadata = Struct({ + magicNumber: u32, + metadata: Enum({ + v0: unsupported, + v1: unsupported, + v2: unsupported, + v3: unsupported, + v4: unsupported, + v5: unsupported, + v6: unsupported, + v7: unsupported, + v8: unsupported, + v9: unsupported, + v10: unsupported, + v11: unsupported, + v12: unsupported, + v13: unsupported, + v14, + v15, + v16, + }), +}) +export type Metadata = CodecType + +const opaqueBytes = Bytes() +const optionOpaque = Option(opaqueBytes) +const opaqueOpaqueBytes = Tuple(compact, opaqueBytes) + +export const decAnyMetadata = ( + input: Uint8Array | HexString, +): CodecType => { + try { + return metadata.dec(input) + } catch (_) {} + + // comes from metadata.metadata_at_version + try { + return metadata.dec(optionOpaque.dec(input)!) + } catch (_) {} + + // comes from state.getMetadata + try { + return metadata.dec(opaqueBytes.dec(input)) + } catch (_) {} + + // comes from metadata.metadata + try { + return metadata.dec(opaqueOpaqueBytes.dec(input)[1]) + } catch (_) {} + + throw null +} diff --git a/packages/bizinikiwi-bindings/src/codecs/metadata/pallets.ts b/packages/bizinikiwi-bindings/src/codecs/metadata/pallets.ts new file mode 100644 index 0000000..c6407b5 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/metadata/pallets.ts @@ -0,0 +1,105 @@ +import { Hex, compactNumber } from "@/codecs/scale" +import { Struct, Option, Vector, u8, str, Enum, _void } from "scale-ts" +import { docs } from "./docs" +import { itemDeprecation, variantDeprecation } from "./deprecation" +import { viewFunction } from "./runtime-api" + +const hashType = Enum({ + Blake2128: _void, + Blake2256: _void, + Blake2128Concat: _void, + Twox128: _void, + Twox256: _void, + Twox64Concat: _void, + Identity: _void, +}) + +const hashers = Vector(hashType) + +export const storageMap = Struct({ + hashers, + key: compactNumber, + value: compactNumber, +}) + +const storageItem = { + name: str, + modifier: u8, + type: Enum({ + plain: compactNumber, + map: storageMap, + }), + fallback: Hex(), + docs, +} + +const enumRefV14 = Option(compactNumber) + +export const v14Pallet = { + name: str, + storage: Option( + Struct({ + prefix: str, + items: Vector(Struct(storageItem)), + }), + ), + calls: enumRefV14, + events: enumRefV14, + constants: Vector( + Struct({ + name: str, + type: compactNumber, + value: Hex(), + docs, + }), + ), + errors: enumRefV14, + index: u8, +} + +export const v15Pallet = { + ...v14Pallet, + docs, +} + +const enumRef = Option( + Struct({ type: compactNumber, deprecationInfo: variantDeprecation }), +) + +export const v16Pallet = { + name: str, + storage: Option( + Struct({ + prefix: str, + items: Vector( + Struct({ + ...storageItem, + deprecationInfo: itemDeprecation, + }), + ), + }), + ), + calls: enumRef, + events: enumRef, + constants: Vector( + Struct({ + name: str, + type: compactNumber, + value: Hex(), + docs, + deprecationInfo: itemDeprecation, + }), + ), + errors: enumRef, + associatedTypes: Vector( + Struct({ + name: str, + type: compactNumber, + docs, + }), + ), + viewFns: Vector(viewFunction), + index: u8, + docs, + deprecationInfo: itemDeprecation, +} diff --git a/packages/bizinikiwi-bindings/src/codecs/metadata/runtime-api.ts b/packages/bizinikiwi-bindings/src/codecs/metadata/runtime-api.ts new file mode 100644 index 0000000..2bb387f --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/metadata/runtime-api.ts @@ -0,0 +1,38 @@ +import { Struct, Vector, str } from "scale-ts" +import { docs } from "./docs" +import { compactNumber, Hex, compactNumber as ty } from "../scale" +import { itemDeprecation } from "./deprecation" + +export const runtimeApiMethod = { + name: str, + inputs: Vector( + Struct({ + name: str, + type: ty, + }), + ), + output: ty, + docs, +} + +export const runtimeApiV15 = Struct({ + name: str, + methods: Vector(Struct(runtimeApiMethod)), + docs, +}) + +export const runtimeApi = Struct({ + name: str, + methods: Vector( + Struct({ ...runtimeApiMethod, deprecationInfo: itemDeprecation }), + ), + docs, + version: compactNumber, + deprecationInfo: itemDeprecation, +}) + +export const viewFunction = Struct({ + id: Hex(32), + ...runtimeApiMethod, + deprecationInfo: itemDeprecation, +}) diff --git a/packages/bizinikiwi-bindings/src/codecs/metadata/unified.ts b/packages/bizinikiwi-bindings/src/codecs/metadata/unified.ts new file mode 100644 index 0000000..e1e8c4f --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/metadata/unified.ts @@ -0,0 +1,161 @@ +import { CodecType } from "scale-ts" +import { V14Lookup } from "./lookup" +import { storageMap } from "./pallets" +import { HexString } from "../scale" +import { itemDeprecation, variantDeprecation } from "./deprecation" +import { viewFunction } from "./runtime-api" +import { V14 } from "./v14" +import { V16 } from "./v16" +import { V15 } from "./v15" +import { Metadata } from "./metadata" + +type EnumRef = + | ({ + type: number + } & (T extends 16 + ? { deprecationInfo: CodecType } + : {})) + | undefined + +type DeprecationInfo = T extends 16 + ? { deprecationInfo: CodecType } + : {} + +export type UnifiedMetadata = { + version: T + lookup: V14Lookup + pallets: Array< + { + name: string + storage: + | { + prefix: string + items: Array< + { + name: string + modifier: number + type: + | { tag: "plain"; value: number } + | { tag: "map"; value: CodecType } + fallback: HexString + docs: string[] + } & DeprecationInfo + > + } + | undefined + calls: EnumRef + events: EnumRef + constants: Array< + { + name: string + type: number + value: HexString + docs: string[] + } & DeprecationInfo + > + errors: EnumRef + associatedTypes: Array<{ + name: string + type: number + docs: string[] + }> + viewFns: Array> + index: number + docs: string[] + } & DeprecationInfo + > + extrinsic: { + version: number[] + signedExtensions: Array<{ + identifier: string + type: number + additionalSigned: number + }> + } & (T extends 14 + ? { + type: number + } + : { address: number; call: number; signature: number }) & + (T extends 16 + ? { signedExtensionsByVersion: Array<[number, number[]]> } + : {}) + apis: Array< + { + name: string + methods: Array< + { + name: string + inputs: Array<{ name: string; type: number }> + output: number + docs: string[] + } & DeprecationInfo + > + docs: string[] + } & (T extends 16 ? { version: number } : {}) & + DeprecationInfo + > +} & (T extends 14 + ? {} + : { + outerEnums: { call: number; event: number; error: number } + custom: Array<[string, { type: number; value: HexString }]> + }) + +export const unifyMetadata = ( + metadata: Metadata | Metadata["metadata"] | V14 | V15 | V16, +): UnifiedMetadata => { + // complete metadata + if ("magicNumber" in metadata) metadata = metadata.metadata + if ("tag" in metadata) { + if ( + metadata.tag !== "v14" && + metadata.tag !== "v15" && + metadata.tag !== "v16" + ) + throw new Error("Only metadata 14, 15, and 16 are supported") + metadata = metadata.value + } + + // v16 + if ("signedExtensionsByVersion" in metadata.extrinsic) { + return { version: 16, ...(metadata as V16) } + } + // v15 + if ("custom" in metadata) { + const { lookup, extrinsic, custom, apis, pallets, outerEnums } = + metadata as V15 + + return { + version: 15, + lookup, + pallets: pallets.map((p): UnifiedMetadata<15>["pallets"][number] => ({ + ...p, + calls: p.calls != null ? { type: p.calls } : undefined, + events: p.events != null ? { type: p.events } : undefined, + errors: p.errors != null ? { type: p.errors } : undefined, + viewFns: [], + associatedTypes: [], + })), + extrinsic: { ...extrinsic, version: [extrinsic.version] }, + apis, + outerEnums, + custom, + } + } + // fallback, v14 + const { lookup, extrinsic, pallets } = metadata as V14 + return { + version: 14, + lookup, + pallets: pallets.map((p): UnifiedMetadata<14>["pallets"][number] => ({ + ...p, + calls: p.calls != null ? { type: p.calls } : undefined, + events: p.events != null ? { type: p.events } : undefined, + errors: p.errors != null ? { type: p.errors } : undefined, + viewFns: [], + associatedTypes: [], + })), + extrinsic: { ...extrinsic, version: [extrinsic.version] }, + apis: [], + } +} diff --git a/packages/bizinikiwi-bindings/src/codecs/metadata/v14.ts b/packages/bizinikiwi-bindings/src/codecs/metadata/v14.ts new file mode 100644 index 0000000..fa3f284 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/metadata/v14.ts @@ -0,0 +1,33 @@ +import { CodecType, Struct, Vector, createCodec, str, u8 } from "scale-ts" +import { lookup } from "./lookup" +import { compactNumber } from "../scale" +import { v14Pallet } from "./pallets" +import { runtimeApi } from "./runtime-api" + +const empty = new Uint8Array() +const Always = (value: T) => + createCodec( + () => empty, + () => value, + ) + +const extrinsic = Struct({ + type: compactNumber, + version: u8, + signedExtensions: Vector( + Struct({ + identifier: str, + type: compactNumber, + additionalSigned: compactNumber, + }), + ), +}) + +export const v14 = Struct({ + lookup, + pallets: Vector(Struct({ ...v14Pallet, docs: Always([] as string[]) })), + extrinsic, + type: compactNumber, + apis: Always([] as Array>), +}) +export type V14 = CodecType diff --git a/packages/bizinikiwi-bindings/src/codecs/metadata/v15.ts b/packages/bizinikiwi-bindings/src/codecs/metadata/v15.ts new file mode 100644 index 0000000..98bc604 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/metadata/v15.ts @@ -0,0 +1,35 @@ +import { CodecType, Struct, Tuple, Vector, str, u8 } from "scale-ts" +import { lookup } from "./lookup" +import { v15Pallet } from "./pallets" +import { Hex, compactNumber as ty } from "../scale" +import { runtimeApiV15 } from "./runtime-api" + +const extrinsic = Struct({ + version: u8, + address: ty, + call: ty, + signature: ty, + extra: ty, + signedExtensions: Vector( + Struct({ + identifier: str, + type: ty, + additionalSigned: ty, + }), + ), +}) + +export const v15 = Struct({ + lookup, + pallets: Vector(Struct(v15Pallet)), + extrinsic, + type: ty, + apis: Vector(runtimeApiV15), + outerEnums: Struct({ + call: ty, + event: ty, + error: ty, + }), + custom: Vector(Tuple(str, Struct({ type: ty, value: Hex() }))), +}) +export type V15 = CodecType diff --git a/packages/bizinikiwi-bindings/src/codecs/metadata/v16.ts b/packages/bizinikiwi-bindings/src/codecs/metadata/v16.ts new file mode 100644 index 0000000..69aff5b --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/metadata/v16.ts @@ -0,0 +1,34 @@ +import { CodecType, Struct, Tuple, Vector, str, u8 } from "scale-ts" +import { lookup } from "./lookup" +import { v16Pallet } from "./pallets" +import { compactNumber, Hex, compactNumber as ty } from "../scale" +import { runtimeApi } from "./runtime-api" + +const extrinsic = Struct({ + version: Vector(u8), + address: ty, + call: ty, + signature: ty, + signedExtensionsByVersion: Vector(Tuple(u8, Vector(compactNumber))), + signedExtensions: Vector( + Struct({ + identifier: str, + type: ty, + additionalSigned: ty, + }), + ), +}) + +export const v16 = Struct({ + lookup, + pallets: Vector(Struct(v16Pallet)), + extrinsic, + apis: Vector(runtimeApi), + outerEnums: Struct({ + call: ty, + event: ty, + error: ty, + }), + custom: Vector(Tuple(str, Struct({ type: ty, value: Hex() }))), +}) +export type V16 = CodecType diff --git a/packages/bizinikiwi-bindings/src/codecs/scale/AccountId.ts b/packages/bizinikiwi-bindings/src/codecs/scale/AccountId.ts new file mode 100644 index 0000000..01d86e7 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/scale/AccountId.ts @@ -0,0 +1,25 @@ +import { Bytes, enhanceCodec } from "scale-ts" +import { + getSs58AddressInfo, + SS58String, + fromBufferToBase58, +} from "@/utils/ss58-util" + +function fromBase58ToBuffer(nBytes: number, _ss58Format: number) { + return (address: SS58String) => { + const info = getSs58AddressInfo(address) + if (!info.isValid) throw new Error("Invalid checksum") + const { publicKey } = info + if (publicKey.length !== nBytes) + throw new Error("Invalid public key length") + + return publicKey + } +} + +export const AccountId = (ss58Format: number = 42, nBytes: 32 | 33 = 32) => + enhanceCodec( + Bytes(nBytes), + fromBase58ToBuffer(nBytes, ss58Format), + fromBufferToBase58(ss58Format), + ) diff --git a/packages/bizinikiwi-bindings/src/codecs/scale/Binary.ts b/packages/bizinikiwi-bindings/src/codecs/scale/Binary.ts new file mode 100644 index 0000000..2c0c0c1 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/scale/Binary.ts @@ -0,0 +1,106 @@ +import { + Bytes, + Codec, + Decoder, + Encoder, + Tuple, + compact, + createCodec, +} from "scale-ts" +import { fromHex, mergeUint8, toHex } from "@pezkuwi/papi-utils" +import type { HexString } from "./Hex" +import { SS58String } from "@/utils" +import { AccountId } from "./AccountId" + +const textEncoder = new TextEncoder() +const textDecoder = new TextDecoder() +const opaqueBytesDec = Tuple(compact, Bytes(Infinity))[1] + +export class Binary { + #bytes: Uint8Array + #opaqueBytes: Uint8Array | null = null + #hex: HexString | null = null + #opaqueHex: HexString | null = null + #str: string | null = null + + constructor(data: Uint8Array, opaque = false) { + if (opaque) { + try { + const [len, bytes] = opaqueBytesDec(data) + if (len === bytes.length) { + this.#bytes = bytes + this.#opaqueBytes = data + return + } + } catch (_) {} + throw new Error("Invalid opaque bytes") + } else this.#bytes = data + } + + asText = () => (this.#str ??= textDecoder.decode(this.#bytes)) + + asHex = () => (this.#hex ??= toHex(this.#bytes)) as `0x${string}` + asOpaqueHex = () => + (this.#opaqueHex ??= toHex(this.asOpaqueBytes())) as `0x${string}` + + asBytes = () => this.#bytes + asOpaqueBytes = () => + (this.#opaqueBytes ??= mergeUint8([ + compact[0](this.#bytes.length), + this.#bytes, + ])) + + static fromText(input: string): Binary { + return new this(textEncoder.encode(input)) + } + + static fromHex(input: HexString): Binary { + return new this(fromHex(input)) + } + static fromOpaqueHex(input: HexString): Binary { + return new this(fromHex(input), true) + } + + static fromBytes(input: Uint8Array): Binary { + return new this(input) + } + static fromOpaqueBytes(input: Uint8Array): Binary { + return new this(input, true) + } +} + +const [accountIdEncoder] = AccountId() +export class FixedSizeBinary<_L extends number> extends Binary { + constructor(data: Uint8Array) { + super(data) + } + + static fromArray & { length: L }>( + input: I, + ) { + return new this(new Uint8Array(input)) + } + + static fromAccountId32( + input: L extends 32 ? SS58String : never, + ) { + return new this(accountIdEncoder(input)) + } +} + +const enc = (nBytes?: number): Encoder => { + const _enc = Bytes.enc(nBytes) + return (value) => _enc(value.asBytes()) +} + +const dec = (nBytes?: number): Decoder => { + const _dec = Bytes.dec(nBytes) + const Bin = nBytes == null ? Binary : FixedSizeBinary + return (value) => Bin.fromBytes(_dec(value)) +} + +export const Bin = (nBytes?: number): Codec => + createCodec(enc(nBytes), dec(nBytes)) + +Bin.enc = enc +Bin.dec = dec diff --git a/packages/bizinikiwi-bindings/src/codecs/scale/BitSeq.ts b/packages/bizinikiwi-bindings/src/codecs/scale/BitSeq.ts new file mode 100644 index 0000000..536f349 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/scale/BitSeq.ts @@ -0,0 +1,53 @@ +import { + Bytes, + Codec, + Decoder, + Encoder, + createCodec, + createDecoder, +} from "scale-ts" +import { compactNumber } from "./compact" +import { mergeUint8 } from "@pezkuwi/papi-utils" + +export type BitSeq = Array<0 | 1> + +const dec: (isLsb?: boolean) => Decoder = (isLsb = true) => + createDecoder((data) => { + const bitsLen = compactNumber.dec(data) + const bytesLen = Math.ceil(bitsLen / 8) + const bytes = Bytes(bytesLen).dec(data) + + const result = new Array<0 | 1>(bitsLen) + let resultIdx = 0 + bytes.forEach((val) => { + for (let i = 0; i < 8 && resultIdx < bitsLen; i++) { + const actualIdx = isLsb ? i : 7 - i + result[resultIdx++] = ((val >> actualIdx) & 1) as 1 | 0 + } + }) + return result + }) + +const enc: (isLsb?: boolean) => Encoder = + (isLsb = true) => + (input) => { + const lenEncoded = compactNumber.enc(input.length) + const nBytes = Math.ceil(input.length / 8) + + const bytes = new Uint8Array(nBytes) + for (let byteIdx = 0; byteIdx < nBytes; byteIdx++) { + let inputIdx = byteIdx * 8 + let byte = 0 + for (let i = 0; i < 8 && inputIdx < input.length; i++, inputIdx++) + byte |= input[inputIdx] << (isLsb ? i : 7 - i) + bytes[byteIdx] = byte + } + + return mergeUint8([lenEncoded, bytes]) + } + +export const BitSeq = (isLsb?: boolean): Codec => + createCodec(enc(isLsb), dec(isLsb)) + +BitSeq.enc = enc +BitSeq.dec = dec diff --git a/packages/bizinikiwi-bindings/src/codecs/scale/Hex.ts b/packages/bizinikiwi-bindings/src/codecs/scale/Hex.ts new file mode 100644 index 0000000..e5e7f6c --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/scale/Hex.ts @@ -0,0 +1,20 @@ +import { fromHex, toHex } from "@pezkuwi/papi-utils" +import { Bytes, Codec, Decoder, Encoder, createCodec } from "scale-ts" + +export type HexString = string & { __hexString?: unknown } + +const enc = (nBytes?: number): Encoder => { + const _enc = Bytes.enc(nBytes) + return (value: string) => _enc(fromHex(value)) +} + +const dec = (nBytes?: number): Decoder => { + const _dec = Bytes.dec(nBytes) + return (value) => toHex(_dec(value)) as HexString +} + +export const Hex = (nBytes?: number): Codec => + createCodec(enc(nBytes), dec(nBytes)) + +Hex.enc = enc +Hex.dec = dec diff --git a/packages/bizinikiwi-bindings/src/codecs/scale/Self.ts b/packages/bizinikiwi-bindings/src/codecs/scale/Self.ts new file mode 100644 index 0000000..32336c3 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/scale/Self.ts @@ -0,0 +1,28 @@ +import { Codec, Decoder, Encoder, createCodec, _void } from "scale-ts" + +export const selfEncoder = (value: () => Encoder): Encoder => { + let cache: Encoder = (x) => { + const encoder = value() + cache = encoder + return encoder(x) + } + + return (x) => cache(x) +} + +export const selfDecoder = (value: () => Decoder): Decoder => { + let cache: Decoder = (x) => { + const decoder = value() + const result = decoder + cache = decoder + return result(x) + } + + return (x) => cache(x) +} + +export const Self = (value: () => Codec): Codec => + createCodec( + selfEncoder(() => value().enc), + selfDecoder(() => value().dec), + ) diff --git a/packages/bizinikiwi-bindings/src/codecs/scale/Variant.ts b/packages/bizinikiwi-bindings/src/codecs/scale/Variant.ts new file mode 100644 index 0000000..5c39342 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/scale/Variant.ts @@ -0,0 +1,149 @@ +import { + Codec, + CodecType, + Decoder, + DecoderType, + Encoder, + EncoderType, + Enum as OEnum, + StringRecord, + createCodec, +} from "scale-ts" +import { mapObject } from "@pezkuwi/papi-utils" +import { Enum } from "@/types/enum" +import { withInner } from "./with-inner" + +type Tuple = readonly [T, ...T[]] & { length: N } + +type Push = [...T, V] + +type UnionToIntersection = (U extends any ? (k: U) => void : never) extends ( + k: infer I, +) => void + ? I + : never + +type LastOf = + UnionToIntersection T : never> extends () => infer R + ? R + : never + +type TuplifyUnion< + T, + L = LastOf, + N = [T] extends [never] ? true : false, +> = true extends N ? [] : Push>, L> + +type RestrictedLenTuple> = Tuple< + T, + TuplifyUnion extends Tuple ? V : 0 +> + +const VariantEnc: >>( + inner: O, + x?: RestrictedLenTuple | undefined, +) => Encoder< + Enum<{ + [K in keyof O]: EncoderType + }> +> & { inner: O } = (...args) => { + const enc = OEnum.enc(...(args as [any, any])) + return withInner((v: any) => enc({ tag: v.type, value: v.value }), args[0]) +} + +const VariantDec: >>( + inner: O, + x?: RestrictedLenTuple | undefined, +) => Decoder< + Enum<{ + [K in keyof O]: DecoderType + }> +> & { inner: O } = (...args) => { + const dec = OEnum.dec(...(args as [any])) + return withInner((v: any) => { + const { tag, value } = dec(v) + return Enum(tag as any, value as any) as any + }, args[0]) +} + +export const Variant: { + >>( + inner: O, + indexes?: RestrictedLenTuple | undefined, + ): Codec< + Enum<{ + [K in keyof O]: CodecType + }> + > & { inner: O } + enc: >>( + inner: O_1, + x?: RestrictedLenTuple | undefined, + ) => Encoder< + Enum<{ + [K in keyof O_1]: EncoderType + }> + > & { inner: O_1 } + dec: >>( + inner: O_2, + x?: RestrictedLenTuple | undefined, + ) => Decoder< + Enum<{ + [K in keyof O_2]: DecoderType + }> + > & { inner: O_2 } +} = (inner, ...args) => + withInner( + createCodec( + VariantEnc( + mapObject(inner, ([encoder]) => encoder) as any, + ...(args as any[]), + ), + VariantDec( + mapObject(inner, ([, decoder]) => decoder) as any, + ...(args as any[]), + ) as any, + ), + inner, + ) as any +Variant.enc = VariantEnc +Variant.dec = VariantDec + +export const ScaleEnum: { + >>( + inner: O, + indexes?: RestrictedLenTuple | undefined, + ): Codec< + { + [K in keyof O]: { + tag: K + value: CodecType + } + }[keyof O] + > & { inner: O } + enc: >>( + inner: O_1, + x?: RestrictedLenTuple | undefined, + ) => Encoder< + { + [K_1 in keyof O_1]: { + tag: K_1 + value: EncoderType + } + }[keyof O_1] + > & { inner: O_1 } + dec: >>( + inner: O_2, + x?: RestrictedLenTuple | undefined, + ) => Decoder< + { + [K_2 in keyof O_2]: { + tag: K_2 + value: DecoderType + } + }[keyof O_2] + > & { inner: O_2 } +} = (inner, ...args) => withInner(OEnum(inner, ...(args as any[])), inner) +ScaleEnum.enc = (inner, ...rest) => + withInner(OEnum.enc(inner, ...(rest as any[])), inner) +ScaleEnum.dec = (inner, ...rest) => + withInner(OEnum.dec(inner, ...(rest as any[])), inner) diff --git a/packages/bizinikiwi-bindings/src/codecs/scale/bitSequence.ts b/packages/bizinikiwi-bindings/src/codecs/scale/bitSequence.ts new file mode 100644 index 0000000..07e7c61 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/scale/bitSequence.ts @@ -0,0 +1,30 @@ +import { Bytes, Decoder, Encoder, createCodec, createDecoder } from "scale-ts" +import { compactNumber } from "./compact" + +export interface BitSequence { + bitsLen: number + bytes: Uint8Array +} + +const bitSequenceDecoder: Decoder = createDecoder((data) => { + const bitsLen = compactNumber.dec(data) + + const bytesLen = Math.ceil(bitsLen / 8) + const bytes = Bytes(bytesLen).dec(data) + return { bytes, bitsLen } +}) + +const bitSequenceEncoder: Encoder = (input) => { + if (input.bitsLen > input.bytes.length * 8) + throw new Error( + `Not enough bytes. (bitsLen:${input.bitsLen}, bytesLen:${input.bytes.length})`, + ) + + const lenEncoded = compactNumber.enc(input.bitsLen) + const result = new Uint8Array(input.bytes.length + lenEncoded.length) + result.set(lenEncoded, 0) + result.set(input.bytes, lenEncoded.length) + return result +} + +export const bitSequence = createCodec(bitSequenceEncoder, bitSequenceDecoder) diff --git a/packages/bizinikiwi-bindings/src/codecs/scale/char.ts b/packages/bizinikiwi-bindings/src/codecs/scale/char.ts new file mode 100644 index 0000000..5c48f49 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/scale/char.ts @@ -0,0 +1,7 @@ +import { enhanceCodec, u8 } from "scale-ts" + +export const char = enhanceCodec( + u8, + (str: string) => str.charCodeAt(0), + String.fromCharCode, +) diff --git a/packages/bizinikiwi-bindings/src/codecs/scale/compact.ts b/packages/bizinikiwi-bindings/src/codecs/scale/compact.ts new file mode 100644 index 0000000..6592200 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/scale/compact.ts @@ -0,0 +1,4 @@ +import { compact, enhanceCodec } from "scale-ts" + +export const compactNumber = enhanceCodec(compact, (v) => v, Number) +export const compactBn = enhanceCodec(compact, (v) => v, BigInt) diff --git a/packages/bizinikiwi-bindings/src/codecs/scale/ethAccount.ts b/packages/bizinikiwi-bindings/src/codecs/scale/ethAccount.ts new file mode 100644 index 0000000..97d898f --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/scale/ethAccount.ts @@ -0,0 +1,39 @@ +import { fromHex, toHex } from "@pezkuwi/papi-utils" +import { Bytes, createCodec, createDecoder } from "scale-ts" +import { Keccak256 } from "../../hashes" +import { textEncoder } from "./fixed-str" + +const getFormattedAddress = (hexAddress: string) => { + const nonChecksum = hexAddress.slice(2) + const hashedAddress = toHex(Keccak256(textEncoder.encode(nonChecksum))).slice( + 2, + ) + const result = new Array(40) + + for (let i = 0; i < 40; i++) { + const checksumVal = parseInt(hashedAddress[i], 16) + const char = nonChecksum[i] + result[i] = checksumVal > 7 ? char.toUpperCase() : char + } + + return `0x${result.join("")}` +} + +const bytes20Dec = Bytes(20)[1] + +export const ethAccount = createCodec( + (input: string) => { + const bytes = fromHex(input) + if (bytes.length !== 20) + throw new Error(`Invalid length found on EthAddress(${input})`) + + const hexAddress = toHex(bytes) + if (input === hexAddress || input === hexAddress.toUpperCase()) return bytes + + if (getFormattedAddress(hexAddress) !== input) + throw new Error(`Invalid checksum found on EthAddress(${input})`) + + return bytes + }, + createDecoder((bytes) => getFormattedAddress(toHex(bytes20Dec(bytes)))), +) diff --git a/packages/bizinikiwi-bindings/src/codecs/scale/fixed-str.ts b/packages/bizinikiwi-bindings/src/codecs/scale/fixed-str.ts new file mode 100644 index 0000000..3a0e7e7 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/scale/fixed-str.ts @@ -0,0 +1,11 @@ +import { Bytes, enhanceCodec } from "scale-ts" + +export const textEncoder = new TextEncoder() +const textDecoder = new TextDecoder() + +export const fixedStr = (nBytes: number) => + enhanceCodec( + Bytes(nBytes), + (str: string) => textEncoder.encode(str), + (bytes) => textDecoder.decode(bytes), + ) diff --git a/packages/bizinikiwi-bindings/src/codecs/scale/shaped.ts b/packages/bizinikiwi-bindings/src/codecs/scale/shaped.ts new file mode 100644 index 0000000..4541cc6 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/scale/shaped.ts @@ -0,0 +1,100 @@ +import { + Struct as OStruct, + Tuple as OTuple, + Vector as OVector, + Result as OResult, + Option as OOption, + Codec, + Encoder, + Decoder, + StringRecord, + CodecType, + EncoderType, + DecoderType, + ResultPayload, +} from "scale-ts" +import { withInner } from "./with-inner" + +export const Struct: { + >>( + codecs: A, + ): Codec<{ [K in keyof A]: CodecType }> & { inner: A } + enc: >>( + encoders: A_1, + ) => Encoder<{ [K_1 in keyof A_1]: EncoderType }> & { inner: A_1 } + dec: >>( + decoders: A_2, + ) => Decoder<{ [K_2 in keyof A_2]: DecoderType }> & { inner: A_2 } +} = (codecs) => withInner(OStruct(codecs), codecs) +Struct.enc = (x) => withInner(OStruct.enc(x), x) +Struct.dec = (x) => withInner(OStruct.dec(x), x) + +export const Tuple: { + []>( + ...inner: A + ): Codec<{ [K in keyof A]: A[K] extends Codec ? D : unknown }> & { + inner: A + } + enc: []>( + ...encoders: A_1 + ) => Encoder<{ + [K_1 in keyof A_1]: A_1[K_1] extends Encoder ? D_1 : unknown + }> & { inner: A_1 } + dec: []>( + ...decoders: A_2 + ) => Decoder<{ + [K_2 in keyof A_2]: A_2[K_2] extends Decoder ? D_2 : unknown + }> & { inner: A_2 } +} = (...inner) => withInner(OTuple(...inner), inner) +Tuple.enc = (...inner) => withInner(OTuple.enc(...inner), inner) +Tuple.dec = (...inner) => withInner(OTuple.dec(...inner), inner) + +export const Vector: { + ( + inner: Codec, + size?: number | undefined, + ): Codec & { inner: Codec } + enc: ( + inner: Encoder, + size?: number | undefined, + ) => Encoder & { inner: Encoder } + dec: ( + getter: Decoder, + size?: number | undefined, + ) => Decoder & { inner: Decoder } +} = (inner, ...rest) => withInner(OVector(inner, ...rest), inner) +Vector.enc = (inner, ...rest) => withInner(OVector.enc(inner, ...rest), inner) +Vector.dec = (inner, ...rest) => withInner(OVector.dec(inner, ...rest), inner) + +export const Result: { + ( + okCodec: Codec, + koCodec: Codec, + ): Codec> & { inner: { ok: Codec; ko: Codec } } + dec: ( + okDecoder: Decoder, + koDecoder: Decoder, + ) => Decoder> & { + inner: { ok: Decoder; ko: Decoder } + } + enc: ( + okEncoder: Encoder, + koEncoder: Encoder, + ) => Encoder> & { + inner: { ok: Encoder; ko: Encoder } + } +} = (ok, ko) => withInner(OResult(ok, ko), { ok, ko }) +Result.enc = (ok, ko) => withInner(OResult.enc(ok, ko), { ok, ko }) +Result.dec = (ok, ko) => withInner(OResult.dec(ok, ko), { ok, ko }) + +export const Option: { + (inner: Codec): Codec & { inner: Codec } + enc: ( + inner: Encoder, + ) => Encoder & { inner: Encoder } + dec: ( + inner: Decoder, + ) => Decoder & { inner: Decoder } +} = (inner) => withInner(OOption(inner), inner) +Option.enc = (inner) => withInner(OOption.enc(inner), inner) +Option.dec = (inner) => withInner(OOption.dec(inner), inner) diff --git a/packages/bizinikiwi-bindings/src/codecs/scale/with-inner.ts b/packages/bizinikiwi-bindings/src/codecs/scale/with-inner.ts new file mode 100644 index 0000000..90aeb7e --- /dev/null +++ b/packages/bizinikiwi-bindings/src/codecs/scale/with-inner.ts @@ -0,0 +1,5 @@ +export const withInner = (codec: T, inner: I): T & { inner: I } => { + const result: T & { inner: I } = codec as any + result.inner = inner + return result +} diff --git a/packages/bizinikiwi-bindings/src/extrinsics/extrinsic-format.ts b/packages/bizinikiwi-bindings/src/extrinsics/extrinsic-format.ts new file mode 100644 index 0000000..c23aaeb --- /dev/null +++ b/packages/bizinikiwi-bindings/src/extrinsics/extrinsic-format.ts @@ -0,0 +1,28 @@ +import { enhanceCodec, u8 } from "scale-ts" + +const TYPES = { + bare: 0b00, + 0b00: "bare", + general: 0b01, + 0b01: "general", + signed: 0b10, + 0b10: "signed", +} as const + +export type ExtrinsicFormat = + | { version: 4; type: "bare" | "signed" } + | { version: 5; type: "bare" | "general" } + +export const extrinsicFormat = enhanceCodec( + u8, + ({ version, type }) => version + (TYPES[type] << 6), + (v) => { + const version = v & 0x3f // 0b0011_1111 + const type = v >> 6 + if (version === 4 && (type === TYPES.bare || type === TYPES.signed)) + return { version, type: TYPES[type] } + if (version === 5 && (type === TYPES.bare || type === TYPES.general)) + return { version, type: TYPES[type] } + throw new Error(`ExtrinsicFormat ${v} not valid`) + }, +) diff --git a/packages/bizinikiwi-bindings/src/hashes/blake2.ts b/packages/bizinikiwi-bindings/src/hashes/blake2.ts new file mode 100644 index 0000000..f1a0eee --- /dev/null +++ b/packages/bizinikiwi-bindings/src/hashes/blake2.ts @@ -0,0 +1,11 @@ +import { mergeUint8 } from "@pezkuwi/papi-utils" +import { blake2b } from "@noble/hashes/blake2.js" + +const len32 = { dkLen: 32 } +export const Blake2256 = (encoded: Uint8Array) => blake2b(encoded, len32) + +const len16 = { dkLen: 16 } +export const Blake2128 = (encoded: Uint8Array) => blake2b(encoded, len16) + +export const Blake2128Concat = (encoded: Uint8Array) => + mergeUint8([Blake2128(encoded), encoded]) diff --git a/packages/bizinikiwi-bindings/src/hashes/blake3.ts b/packages/bizinikiwi-bindings/src/hashes/blake3.ts new file mode 100644 index 0000000..900f73e --- /dev/null +++ b/packages/bizinikiwi-bindings/src/hashes/blake3.ts @@ -0,0 +1,8 @@ +import { mergeUint8 } from "@pezkuwi/papi-utils" +import { blake3 } from "@noble/hashes/blake3.js" + +const len32 = { dkLen: 32 } +export const Blake3256 = (encoded: Uint8Array) => blake3(encoded, len32) + +export const Blake3256Concat = (encoded: Uint8Array) => + mergeUint8([Blake3256(encoded), encoded]) diff --git a/packages/bizinikiwi-bindings/src/hashes/h64.ts b/packages/bizinikiwi-bindings/src/hashes/h64.ts new file mode 100644 index 0000000..eab812b --- /dev/null +++ b/packages/bizinikiwi-bindings/src/hashes/h64.ts @@ -0,0 +1,177 @@ +// Adapted implementation based on: https://github.com/pierrec/js-xxhash/blob/7ff5ced282f97dba121109d7013e0fa80360398c/lib/xxhash64.js + +// Distributed under the MIT software license, see the accompanying +// file LICENSE or http://www.opensource.org/licenses/mit-license.php. + +// helper functions +const bigintFromU16 = ( + v0: number, + v1: number, + v2: number, + v3: number, +): bigint => + new DataView(new Uint16Array([v0, v1, v2, v3]).buffer).getBigUint64(0, true) + +const MASK_64 = 2n ** 64n - 1n + +const rotl = (input: bigint, nBits: bigint) => + ((input << nBits) & MASK_64) | (input >> (64n - nBits)) + +const multiply = (a: bigint, b: bigint) => (a * b) & MASK_64 + +const add = (a: bigint, b: bigint) => (a + b) & MASK_64 + +// constants +const PRIME64_1 = 11400714785074694791n +const PRIME64_2 = 14029467366897019727n +const PRIME64_3 = 1609587929392839161n +const PRIME64_4 = 9650029242287828579n +const PRIME64_5 = 2870177450012600261n + +export function h64(input: Uint8Array, seed: bigint = 0n) { + let v1 = add(add(seed, PRIME64_1), PRIME64_2) + let v2 = add(seed, PRIME64_2) + let v3 = seed + let v4 = seed - PRIME64_1 + let totalLen = input.length + let memsize = 0 + let memory: Uint8Array | null = null + + ;(function update() { + let p = 0 + let bEnd = p + totalLen + + if (!totalLen) return + + memory = new Uint8Array(32) + + if (totalLen < 32) { + memory.set(input.subarray(0, totalLen), memsize) + + memsize += totalLen + return + } + + if (p <= bEnd - 32) { + const limit = bEnd - 32 + + do { + let other + other = bigintFromU16( + (input[p + 1] << 8) | input[p], + (input[p + 3] << 8) | input[p + 2], + (input[p + 5] << 8) | input[p + 4], + (input[p + 7] << 8) | input[p + 6], + ) + v1 = multiply(rotl(add(v1, multiply(other, PRIME64_2)), 31n), PRIME64_1) + p += 8 + other = bigintFromU16( + (input[p + 1] << 8) | input[p], + (input[p + 3] << 8) | input[p + 2], + (input[p + 5] << 8) | input[p + 4], + (input[p + 7] << 8) | input[p + 6], + ) + + v2 = multiply(rotl(add(v2, multiply(other, PRIME64_2)), 31n), PRIME64_1) + p += 8 + other = bigintFromU16( + (input[p + 1] << 8) | input[p], + (input[p + 3] << 8) | input[p + 2], + (input[p + 5] << 8) | input[p + 4], + (input[p + 7] << 8) | input[p + 6], + ) + + v3 = multiply(rotl(add(v3, multiply(other, PRIME64_2)), 31n), PRIME64_1) + p += 8 + other = bigintFromU16( + (input[p + 1] << 8) | input[p], + (input[p + 3] << 8) | input[p + 2], + (input[p + 5] << 8) | input[p + 4], + (input[p + 7] << 8) | input[p + 6], + ) + v4 = multiply(rotl(add(v4, multiply(other, PRIME64_2)), 31n), PRIME64_1) + p += 8 + } while (p <= limit) + } + + if (p < bEnd) { + memory.set(input.subarray(p, bEnd), memsize) + memsize = bEnd - p + } + })() + + input = memory || input + + let result: bigint + let p = 0 + + if (totalLen >= 32) { + result = rotl(v1, 1n) + result = add(result, rotl(v2, 7n)) + result = add(result, rotl(v3, 12n)) + result = add(result, rotl(v4, 18n)) + + v1 = multiply(rotl(multiply(v1, PRIME64_2), 31n), PRIME64_1) + result = result ^ v1 + result = add(multiply(result, PRIME64_1), PRIME64_4) + + v2 = multiply(rotl(multiply(v2, PRIME64_2), 31n), PRIME64_1) + result = result ^ v2 + result = add(multiply(result, PRIME64_1), PRIME64_4) + + v3 = multiply(rotl(multiply(v3, PRIME64_2), 31n), PRIME64_1) + result = result ^ v3 + result = add(multiply(result, PRIME64_1), PRIME64_4) + + v4 = multiply(rotl(multiply(v4, PRIME64_2), 31n), PRIME64_1) + result = result ^ v4 + result = add(multiply(result, PRIME64_1), PRIME64_4) + } else { + result = add(seed, PRIME64_5) + } + + result = add(result, BigInt(totalLen)) + + while (p <= memsize - 8) { + let temp = bigintFromU16( + (input[p + 1] << 8) | input[p], + (input[p + 3] << 8) | input[p + 2], + (input[p + 5] << 8) | input[p + 4], + (input[p + 7] << 8) | input[p + 6], + ) + temp = multiply(rotl(multiply(temp, PRIME64_2), 31n), PRIME64_1) + result = add(multiply(rotl(result ^ temp, 27n), PRIME64_1), PRIME64_4) + p += 8 + } + + if (p + 4 <= memsize) { + let temp = multiply( + bigintFromU16( + (input[p + 1] << 8) | input[p], + (input[p + 3] << 8) | input[p + 2], + 0, + 0, + ), + PRIME64_1, + ) + + result = add(multiply(rotl(result ^ temp, 23n), PRIME64_2), PRIME64_3) + p += 4 + } + + while (p < memsize) { + const temp = multiply(bigintFromU16(input[p++], 0, 0, 0), PRIME64_5) + result = multiply(rotl(result ^ temp, 11n), PRIME64_1) + } + + let temp = result >> 33n + result = multiply(result ^ temp, PRIME64_2) + + temp = result >> 29n + result = multiply(result ^ temp, PRIME64_3) + + temp = result >> 32n + result ^= temp + + return result +} diff --git a/packages/bizinikiwi-bindings/src/hashes/identity.ts b/packages/bizinikiwi-bindings/src/hashes/identity.ts new file mode 100644 index 0000000..7730eed --- /dev/null +++ b/packages/bizinikiwi-bindings/src/hashes/identity.ts @@ -0,0 +1 @@ +export const Identity = (encoded: Uint8Array): Uint8Array => encoded diff --git a/packages/bizinikiwi-bindings/src/hashes/keccak.ts b/packages/bizinikiwi-bindings/src/hashes/keccak.ts new file mode 100644 index 0000000..8ceaacc --- /dev/null +++ b/packages/bizinikiwi-bindings/src/hashes/keccak.ts @@ -0,0 +1,3 @@ +import { keccak_256 } from "@noble/hashes/sha3.js" + +export const Keccak256: (input: Uint8Array) => Uint8Array = keccak_256 diff --git a/packages/bizinikiwi-bindings/src/hashes/twoX.ts b/packages/bizinikiwi-bindings/src/hashes/twoX.ts new file mode 100644 index 0000000..c6ed5e8 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/hashes/twoX.ts @@ -0,0 +1,28 @@ +import { mergeUint8 } from "@pezkuwi/papi-utils" +import { u64 } from "scale-ts" +import { h64 } from "./h64" + +export const Twox128 = (input: Uint8Array): Uint8Array => { + const result = new Uint8Array(16) + const dv = new DataView(result.buffer) + + dv.setBigUint64(0, h64(input), true) + dv.setBigUint64(8, h64(input, 1n), true) + + return result +} + +export const Twox256 = (input: Uint8Array): Uint8Array => { + const result = new Uint8Array(32) + const dv = new DataView(result.buffer) + + dv.setBigUint64(0, h64(input), true) + dv.setBigUint64(8, h64(input, 1n), true) + dv.setBigUint64(16, h64(input, 2n), true) + dv.setBigUint64(24, h64(input, 3n), true) + + return result +} + +export const Twox64Concat = (encoded: Uint8Array): Uint8Array => + mergeUint8([u64.enc(h64(encoded)), encoded]) diff --git a/packages/bizinikiwi-bindings/src/storage.ts b/packages/bizinikiwi-bindings/src/storage.ts new file mode 100644 index 0000000..e5265e5 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/storage.ts @@ -0,0 +1,106 @@ +import { fromHex, mergeUint8, toHex } from "@pezkuwi/papi-utils" +import type { Codec } from "scale-ts" +import { + Blake2128, + Blake2128Concat, + Blake2256, + Identity, + Twox128, + Twox256, + Twox64Concat, +} from "./hashes" + +export type EncoderWithHash = [Codec, (input: Uint8Array) => Uint8Array] + +const textEncoder = new TextEncoder() + +// the value indicates: +// - when positive: the number of bytes to skip before reaching the transparent-encoded key +// - when negative: the number of bytes that the opaque hasher will generate +const hashers: Map<(input: Uint8Array) => Uint8Array, number> = new Map([ + [Identity, 0], + [Twox64Concat, 8], + [Blake2128Concat, 16], + [Blake2128, -16], + [Blake2256, -32], + [Twox128, -16], + [Twox256, -32], +]) + +export type OpaqueKeyHash = string & { __opaqueKeyHash?: unknown } + +export const Storage = (pallet: string) => { + const palledEncoded = Twox128(textEncoder.encode(pallet)) + return >>( + name: string, + ...encoders: [...A] + ): { + enc: ( + ...args: { + [K in keyof A]: A[K] extends EncoderWithHash ? V : unknown + } + ) => string + dec: (value: string) => { + [K in keyof A]: A[K] extends EncoderWithHash ? V : unknown + } + } => { + const palletItemEncoded = mergeUint8([ + palledEncoded, + Twox128(textEncoder.encode(name)), + ]) + + const palletItemEncodedHex = toHex(palletItemEncoded) + + const dec = ( + key: string, + ): { + [K in keyof A]: A[K] extends EncoderWithHash ? V : unknown + } => { + if (!key.startsWith(palletItemEncodedHex)) + throw new Error(`key does not match this storage (${pallet}.${name})`) + + if (encoders.length === 0) return [] as any + + const argsKey = fromHex(key.slice(palletItemEncodedHex.length)) + const result = new Array(encoders.length) + for (let i = 0, cur = 0; i < encoders.length; i++) { + const [codec, hasher] = encoders[i] + const hBytes = hashers.get(hasher) + if (hBytes == null) throw new Error("Unknown hasher") + if (hBytes < 0) { + const opaqueBytes = hBytes * -1 + result[i] = toHex(argsKey.slice(cur, cur + opaqueBytes)) + cur += opaqueBytes + } else { + cur += hBytes + result[i] = codec.dec(argsKey.slice(cur)) + cur += codec.enc(result[i]).length + } + } + return result as any + } + + const fns = encoders.map( + ([{ enc }, hash]) => + (val: any) => + hash(enc(val)), + ) + + const enc = ( + ...args: { + [K in keyof A]: A[K] extends EncoderWithHash ? V : unknown + } + ): string => + toHex( + mergeUint8([ + palletItemEncoded, + ...args.map((val, idx) => fns[idx](val)), + ]), + ) + + return { + enc, + dec, + } + } +} diff --git a/packages/bizinikiwi-bindings/src/trie/node-decoder.ts b/packages/bizinikiwi-bindings/src/trie/node-decoder.ts new file mode 100644 index 0000000..1ee11f9 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/trie/node-decoder.ts @@ -0,0 +1,70 @@ +import { createDecoder, Hex, u16, u8, type HexString } from "../codecs" +import { type TrieNodeHeaderKey, TrieNodeHeaders, type TrieNode } from "./types" + +const varHex = Hex().dec +const allHex = Hex(Infinity).dec +const hex32 = Hex(32).dec +const byte = u8.dec + +const getHeader = ( + bytes: Uint8Array, +): { type: TrieNodeHeaderKey; partialKey: string } => { + const firstByte = byte(bytes) + + let bitsLeft = 6 + const typeId = firstByte >> bitsLeft + const type: TrieNodeHeaderKey = typeId + ? typeId === 1 + ? TrieNodeHeaders.Leaf + : typeId === 2 + ? TrieNodeHeaders.Branch + : TrieNodeHeaders.BranchWithVal + : firstByte >> --bitsLeft + ? TrieNodeHeaders.LeafWithHash + : firstByte >> --bitsLeft + ? TrieNodeHeaders.BranchWithHash + : firstByte + ? TrieNodeHeaders.Reserved + : TrieNodeHeaders.Empty + + let nNibles = firstByte & (0xff >> (8 - bitsLeft)) + if (nNibles === 2 ** bitsLeft - 1) { + let current: number + do nNibles += current = byte(bytes) + while (current === 255) + } + + return { + type, + partialKey: Hex(Math.ceil(nNibles / 2)) + .dec(bytes) + .slice(nNibles % 2 ? 3 : 2), + } +} + +export const trieNodeDec = createDecoder((bytes): TrieNode => { + const header = getHeader(bytes) + const { type } = header + + if (type === "Empty" || type === "Reserved") return header as TrieNode + if (type === "Leaf" || type === "LeafWithHash") + return { + ...header, + value: allHex(bytes), + } as TrieNode + + const bitmap = u16.dec(bytes) + const keys: string[] = [] + for (let i = 0; i < 16; i++) if ((bitmap >> i) & 1) keys.push(i.toString(16)) + + let value: null | HexString = null + if (type === "BranchWithVal") value = varHex(bytes) + if (type === "BranchWithHash") value = hex32(bytes) + + const result: any = { + ...header, + children: Object.fromEntries(keys.map((key) => [key, varHex(bytes)])), + } + if (value !== null) result.value = value + return result +}) diff --git a/packages/bizinikiwi-bindings/src/trie/proofs.ts b/packages/bizinikiwi-bindings/src/trie/proofs.ts new file mode 100644 index 0000000..495d86f --- /dev/null +++ b/packages/bizinikiwi-bindings/src/trie/proofs.ts @@ -0,0 +1,72 @@ +import { toHex } from "@pezkuwi/papi-utils" +import { createDecoder, type HexString } from "../codecs" +import { trieNodeDec } from "./node-decoder" +import { Blake2256 } from "@/hashes" +import { type ProofTrieNode } from "./types" + +export const TrieNodeWithHash = (hasher: (input: Uint8Array) => Uint8Array) => + createDecoder((input): ProofTrieNode => { + const hash = toHex(hasher(new Uint8Array(input.buffer))) + try { + return { + hash, + ...trieNodeDec(input), + } + } catch { + // Sometimes the proofs include random raw-values which are not trie-nodes + return { type: "Raw", hash, value: "" } + } + }) + +export const validateProofs = ( + proofs: Array, + hasher: (input: Uint8Array) => Uint8Array = Blake2256, +): { rootHash: HexString; proofs: Record } | null => { + const proofsList = proofs.map(TrieNodeWithHash(hasher)) + const proofsRecord = Object.fromEntries(proofsList.map((p) => [p.hash, p])) + const hashes = proofsList.map((p) => p.hash) + const roots = new Set(hashes) + + const setRawValue = (input: { + type: "Raw" + hash: HexString + value: HexString + }) => { + if (input.value) return + const val = proofs[hashes.indexOf(input.hash)!] + input.value = typeof val === "string" ? val : toHex(val) + } + + proofsList.forEach((p) => { + if ("children" in p) { + Object.values(p.children).forEach((hash) => { + const child = proofsRecord[hash] + if (child) { + child.parent = p.hash + roots.delete(hash) + } + }) + } + + if (p.type === "BranchWithHash" || p.type === "LeafWithHash") { + const childHash = p.value + const child = proofsRecord[childHash] + if (!child) return + + roots.delete(childHash) + if (child.type !== "Raw") { + Object.keys(child).forEach((k) => delete (child as any)[k]) + ;(child as any).type = "Raw" + child.hash = childHash + } + child.parent = p.hash + setRawValue(child as any) + } + + if (p.type === "Raw") setRawValue(p) + }) + + return roots.size === 1 + ? { rootHash: roots.values().next().value!, proofs: proofsRecord } + : null +} diff --git a/packages/bizinikiwi-bindings/src/trie/types.ts b/packages/bizinikiwi-bindings/src/trie/types.ts new file mode 100644 index 0000000..2eeb00a --- /dev/null +++ b/packages/bizinikiwi-bindings/src/trie/types.ts @@ -0,0 +1,55 @@ +import { type HexString } from "../codecs" + +export const TrieNodeHeaders = { + Leaf: "Leaf", + Branch: "Branch", + BranchWithVal: "BranchWithVal", + LeafWithHash: "LeafWithHash", + BranchWithHash: "BranchWithHash", + Empty: "Empty", + Reserved: "Reserved", +} as const +type TrieNodeHeaders = typeof TrieNodeHeaders +export type TrieNodeHeaderKey = + (typeof TrieNodeHeaders)[keyof typeof TrieNodeHeaders] + +export type Nibble = + | "0" + | "1" + | "2" + | "3" + | "4" + | "5" + | "6" + | "7" + | "8" + | "9" + | "a" + | "b" + | "c" + | "d" + | "e" + | "f" + +export type TrieNode = { partialKey: string } & ( + | { + type: TrieNodeHeaders["Empty"] | TrieNodeHeaders["Reserved"] + } + | { + type: TrieNodeHeaders["Leaf"] | TrieNodeHeaders["LeafWithHash"] + value: HexString + } + | ({ children: Record } & ( + | { type: TrieNodeHeaders["Branch"] } + | { + type: + | TrieNodeHeaders["BranchWithHash"] + | TrieNodeHeaders["BranchWithVal"] + value: HexString + } + )) +) +export type ProofTrieNode = { + hash: HexString + parent?: HexString +} & (TrieNode | { type: "Raw"; value: HexString }) diff --git a/packages/bizinikiwi-bindings/src/types/enum.ts b/packages/bizinikiwi-bindings/src/types/enum.ts new file mode 100644 index 0000000..c95cb1f --- /dev/null +++ b/packages/bizinikiwi-bindings/src/types/enum.ts @@ -0,0 +1,93 @@ +export type Enum = { + [K in keyof T & string]: { + type: K + value: T[K] + } +}[keyof T & string] + +export type EnumVariant< + T extends { type: string; value?: any }, + K extends T["type"], +> = T & { + type: K +} + +export type ExtractEnumValue< + T extends { type: string; value?: any }, + K extends string, +> = EnumVariant["value"] + +type ValueArg = undefined extends V ? [value?: V] : [value: V] + +interface Discriminant { + is( + value: T, + type: K, + ): value is T & { type: K } + as( + value: T, + type: K, + ): ExtractEnumValue +} +const discriminant: Discriminant = { + is( + value: T, + type: K, + ): value is T & { type: K } { + return value.type === type + }, + as(value, type) { + if (type !== value.type) + throw new Error( + `Enum.as(enum, ${type}) used with actual type ${value.type}`, + ) + return value + }, +} +interface EnumFn extends Discriminant { + ( + type: K, + ...[value]: ValueArg> + ): EnumVariant +} +export const Enum: EnumFn = Object.assign((type: string, value?: any) => { + return { + type, + value, + } as any +}, discriminant) + +// well-known enums +export type GetEnum> = { + [K in T["type"]]: ( + ...args: ExtractEnumValue extends undefined + ? [] + : [value: ExtractEnumValue] + ) => EnumVariant +} +export const _Enum = new Proxy( + {}, + { + get(_, prop: string) { + return (value: string) => Enum(prop, value) + }, + }, +) + +// type Bar = Enum<{ +// Kaka: 1 +// Bar: 2 +// }> + +// type FooInput = Enum<{ +// foo: "foo" | undefined +// bar: Bar +// baz: number +// wtf: boolean +// }> + +// declare function foo(foo: FooInput): void +// foo(Enum("bar", Enum("Bar", 2))) + +// const InputEnum: GetEnum = null as any; +// InputEnum.bar(Enum('Bar', 2)) diff --git a/packages/bizinikiwi-bindings/src/utils/multisig.ts b/packages/bizinikiwi-bindings/src/utils/multisig.ts new file mode 100644 index 0000000..caa794d --- /dev/null +++ b/packages/bizinikiwi-bindings/src/utils/multisig.ts @@ -0,0 +1,34 @@ +import { mergeUint8 } from "@pezkuwi/papi-utils" +import { Binary, compact, u16 } from "../codecs" +import { Blake2256 } from "../hashes" + +const PREFIX = Binary.fromText("modlpy/utilisuba").asBytes() +export const getMultisigAccountId = ({ + threshold, + signatories, +}: { + threshold: number + signatories: Uint8Array[] +}) => { + const sortedSignatories = sortMultisigSignatories(signatories) + const payload = mergeUint8([ + PREFIX, + compact.enc(sortedSignatories.length), + ...sortedSignatories, + u16.enc(threshold), + ]) + return Blake2256(payload) +} + +export const sortMultisigSignatories = (signatories: Uint8Array[]) => + signatories.slice().sort((a, b) => { + for (let i = 0; ; i++) { + const overA = i >= a.length + const overB = i >= b.length + + if (overA && overB) return 0 + else if (overA) return -1 + else if (overB) return 1 + else if (a[i] !== b[i]) return a[i] > b[i] ? 1 : -1 + } + }) diff --git a/packages/bizinikiwi-bindings/src/utils/ss58-util.ts b/packages/bizinikiwi-bindings/src/utils/ss58-util.ts new file mode 100644 index 0000000..d411377 --- /dev/null +++ b/packages/bizinikiwi-bindings/src/utils/ss58-util.ts @@ -0,0 +1,93 @@ +import { base58 } from "@scure/base" +import { blake2b } from "@noble/hashes/blake2.js" + +const SS58_PREFIX = new TextEncoder().encode("SS58PRE") +const CHECKSUM_LENGTH = 2 + +export type SS58String = string & { __SS58String?: unknown } +export type SS58AddressInfo = + | { isValid: false } + | { isValid: true; ss58Format: number; publicKey: Uint8Array } + +export const getSs58AddressInfo = (address: SS58String): SS58AddressInfo => { + try { + const decoded = base58.decode(address) + const prefixBytes = decoded.subarray(0, decoded[0] & 0b0100_0000 ? 2 : 1) + const publicKey = decoded.subarray( + prefixBytes.length, + decoded.length - CHECKSUM_LENGTH, + ) + + const checksum = decoded.subarray(prefixBytes.length + publicKey.length) + const expectedChecksum = blake2b( + Uint8Array.of(...SS58_PREFIX, ...prefixBytes, ...publicKey), + { + dkLen: 64, + }, + ).subarray(0, CHECKSUM_LENGTH) + + const isChecksumValid = + checksum[0] === expectedChecksum[0] && checksum[1] === expectedChecksum[1] + + if (!isChecksumValid) return { isValid: false } + + return { + isValid: true, + ss58Format: prefixBytesToNumber(prefixBytes), + publicKey: publicKey.slice(), + } + } catch (_) { + return { isValid: false } + } +} + +const prefixBytesToNumber = (bytes: Uint8Array) => { + const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + return dv.byteLength === 1 ? dv.getUint8(0) : dv.getUint16(0) +} + +const withSs58Cache = (fn: (publicKey: Uint8Array) => SS58String) => { + let cache: Record = {} + let activityCount = 0 + let latestCount = 0 + const checkActivity = () => { + if (activityCount === latestCount) { + cache = {} + activityCount = latestCount = 0 + } else { + latestCount = activityCount + setTimeout(checkActivity, 0) + } + } + + return (publicKey: Uint8Array): SS58String => { + if (++activityCount === 1) checkActivity() + + let entry = cache + const lastIdx = publicKey.length - 1 + for (let i = 0; i <= lastIdx; i++) entry = entry[publicKey[i]] ||= {} + return (entry[publicKey[lastIdx]] ||= fn(publicKey)) + } +} + +export const fromBufferToBase58 = (ss58Format: number) => { + const prefixBytes = + ss58Format < 64 + ? Uint8Array.of(ss58Format) + : Uint8Array.of( + ((ss58Format & 0b0000_0000_1111_1100) >> 2) | 0b0100_0000, + (ss58Format >> 8) | ((ss58Format & 0b0000_0000_0000_0011) << 6), + ) + + return withSs58Cache((publicKey: Uint8Array): SS58String => { + const checksum = blake2b( + Uint8Array.of(...SS58_PREFIX, ...prefixBytes, ...publicKey), + { + dkLen: 64, + }, + ).subarray(0, CHECKSUM_LENGTH) + return base58.encode( + Uint8Array.of(...prefixBytes, ...publicKey, ...checksum), + ) + }) +} diff --git a/packages/merkleize-metadata/README.md b/packages/merkleize-metadata/README.md index d5ade61..8297e16 100644 --- a/packages/merkleize-metadata/README.md +++ b/packages/merkleize-metadata/README.md @@ -1,4 +1,15 @@ -# @polkadot-api/merkleize-metadata +# @pezkuwi/merkleize-metadata + +**Source:** `src/` is [`@polkadot-api/merkleize-metadata`](https://www.npmjs.com/package/@polkadot-api/merkleize-metadata/v/1.1.22) 1.1.22 from +[polkadot-api](https://github.com/polkadot-api/polkadot-api) (MIT), with imports of +`@polkadot-api/*` renamed to `@pezkuwi/*` (`substrate-bindings` → `bizinikiwi-bindings`). `dist/` is its build; +`scripts/check-papi-sources.mjs` fails CI unless the source embedded in `dist/` equals `src/`. + +The upstream README follows. + +--- + +## @polkadot-api/merkleize-metadata This TS package provides utils for the merkleization of [`frame_metadata`](https://docs.rs/frame-metadata/latest/frame_metadata/) as described in [RFC78](https://polkadot-fellows.github.io/RFCs/approved/0078-merkleized-metadata.html). diff --git a/packages/merkleize-metadata/src/codecs.ts b/packages/merkleize-metadata/src/codecs.ts new file mode 100644 index 0000000..f022fce --- /dev/null +++ b/packages/merkleize-metadata/src/codecs.ts @@ -0,0 +1,152 @@ +import { + Bytes, + Codec, + CodecType, + Option, + ScaleEnum, + Struct, + Tuple, + V14Lookup, + Vector, + _void, + bool, + compact, + compactNumber, + extrinsicFormat, + str, + u16, + u32, + u8, +} from "@pezkuwi/bizinikiwi-bindings" + +const extraInfoInner = { + specVersion: u32, + specName: str, + base58Prefix: u16, + decimals: u8, + tokenSymbol: str, +} +export const extraInfo = Struct(extraInfoInner) +export type ExtraInfo = CodecType + +export const hash = Bytes(32) +export const metadataDigest = ScaleEnum({ + V0: _void, + V1: Struct({ + typeInformationTreeRoot: hash, + extrinsicMetadataHash: hash, + ...extraInfoInner, + }), +}) +export type MetadataDigest = CodecType + +export const scalePrimitive = ScaleEnum({ + bool: _void, + char: _void, + str: _void, + u8: _void, + u16: _void, + u32: _void, + u64: _void, + u128: _void, + u256: _void, + i8: _void, + i16: _void, + i32: _void, + i64: _void, + i128: _void, + i256: _void, +}) +export type ScalePrimitive = CodecType + +export const typeRef = ScaleEnum({ + bool: _void, + char: _void, + str: _void, + u8: _void, + u16: _void, + u32: _void, + u64: _void, + u128: _void, + u256: _void, + i8: _void, + i16: _void, + i32: _void, + i64: _void, + i128: _void, + i256: _void, + compactU8: _void, + compactU16: _void, + compactU32: _void, + compactU64: _void, + compactU128: _void, + compactU256: _void, + void: _void, + perId: compactNumber, +}) +export type TypeRef = CodecType + +const field = Struct({ + name: Option(str), + ty: typeRef, + typeName: Option(str), +}) +export const typeDef = ScaleEnum({ + composite: Vector(field), + enumeration: Struct({ + name: str, + fields: Vector(field), + index: compactNumber, + }), + sequence: typeRef, + array: Struct({ + len: u32, + typeParam: typeRef, + }), + tuple: Vector(typeRef), + bitSequence: Struct({ + numBytes: u8, + leastSignificantBitFirst: bool, + }), +}) +export type TypeDef = CodecType + +export const lookupType = Struct({ + path: Vector(str), + typeDef: typeDef, + typeId: compactNumber, +}) +export type LookupEntry = CodecType +export type Lookup = Array +export const lookup: Codec = Vector(lookupType) + +export const extrinsicMetadata = Struct({ + version: u8, + addressTy: typeRef, + callTy: typeRef, + signatureTy: typeRef, + signedExtensions: Vector( + Struct({ + identifier: str, + includedInExtrinsic: typeRef, + includedInSignedData: typeRef, + }), + ), +}) +export type ExtrinsicMetadata = CodecType +export type LookupValue = V14Lookup[number] + +export const extrinsicDec = Tuple.dec( + compact[1], + extrinsicFormat[1], + Bytes(Infinity)[1], +) + +export const proof = Struct({ + leaves: lookup, + leafIdxs: Vector(u32), + proofs: Vector(hash), + extrinsic: extrinsicMetadata, + info: extraInfo, +}) +export type Proof = CodecType diff --git a/packages/merkleize-metadata/src/decode-and-collect.ts b/packages/merkleize-metadata/src/decode-and-collect.ts new file mode 100644 index 0000000..bc53f4d --- /dev/null +++ b/packages/merkleize-metadata/src/decode-and-collect.ts @@ -0,0 +1,135 @@ +import { + HexString, + _void, + compact, + createDecoder, + i128, + i16, + i256, + i32, + i64, + i8, + str, + u128, + u16, + u256, + u32, + u64, + u8, +} from "@pezkuwi/bizinikiwi-bindings" +import { Lookup, TypeDef, TypeRef } from "./codecs" + +const typeRefDecoders = { + bool: u8, + char: u8, + str, + u8, + u16, + u32, + u64, + u128, + u256, + i8, + i16, + i32, + i64, + i128, + i256, + void: _void, + compactU8: compact, + compactU16: compact, + compactU32: compact, + compactU64: compact, + compactU128: compact, + compactU256: compact, +} + +const innerDecodeAndCollect = ( + input: Uint8Array, + typeRef: TypeRef, + idToLookups: Map>, + lookup: Lookup, + collected: Set, +) => { + if (typeRef.tag !== "perId") { + typeRefDecoders[typeRef.tag][1](input) + return + } + + const handleTypeRef = (typeRef: TypeRef) => { + innerDecodeAndCollect(input, typeRef, idToLookups, lookup, collected) + } + + const lookupIdxs = idToLookups.get(typeRef.value)! + const [currentIdx] = lookupIdxs + const current = lookup[currentIdx] + + if (lookupIdxs.length === 1) collected.add(currentIdx) + + switch (current.typeDef.tag) { + case "enumeration": { + const selectedIdx = u8.dec(input) + const [selected, collectedIdx] = lookupIdxs + .map( + (lookupIdx) => + [lookup[lookupIdx].typeDef, lookupIdx] as [ + TypeDef & { tag: "enumeration" }, + number, + ], + ) + .find(([x]) => x.value.index === selectedIdx)! + collected.add(collectedIdx) + selected.value.fields.forEach(({ ty }) => { + handleTypeRef(ty) + }) + break + } + case "sequence": { + const len = compact.dec(input) + for (let i = 0; i < len; i++) handleTypeRef(current.typeDef.value) + break + } + case "array": { + for (let i = 0; i < current.typeDef.value.len; i++) + handleTypeRef(current.typeDef.value.typeParam) + break + } + case "composite": { + current.typeDef.value.forEach((x) => { + handleTypeRef(x.ty) + }) + break + } + case "tuple": { + current.typeDef.value.forEach(handleTypeRef) + break + } + case "bitSequence": + throw new Error("bitSequence is not supported") + } +} + +export const decodeAndCollectKnownLeafs = ( + data: Uint8Array | HexString, + typeRefs: Array, + lookup: Lookup, +): Array => { + let input: Uint8Array = new Uint8Array() + createDecoder((_input) => { + input = _input + })(data) + + const idToLookups = new Map() + lookup.forEach((lookup, idx) => { + const arr = idToLookups.get(lookup.typeId) + if (arr) arr.push(idx) + else idToLookups.set(lookup.typeId, [idx]) + }) + + const result = new Set() + typeRefs.forEach((typeRef) => { + innerDecodeAndCollect(input, typeRef, idToLookups, lookup, result) + }) + + return [...result].sort((a, b) => a - b) +} diff --git a/packages/merkleize-metadata/src/get-accessible-types.ts b/packages/merkleize-metadata/src/get-accessible-types.ts new file mode 100644 index 0000000..2dc83a8 --- /dev/null +++ b/packages/merkleize-metadata/src/get-accessible-types.ts @@ -0,0 +1,60 @@ +import { UnifiedMetadata } from "@pezkuwi/bizinikiwi-bindings" +import { LookupValue } from "./codecs" + +export const getAccessibleTypes = ( + metadata: UnifiedMetadata<15 | 16>, + definitions: Map, +): Map => { + const types = new Set() + + const collectTypesFromId = (id: number) => { + if (types.has(id)) return + + const { tag, value } = definitions.get(id)!.def + switch (tag) { + case "composite": + if (!value.length) break + types.add(id) + value.forEach(({ type }) => { + collectTypesFromId(type) + }) + break + case "variant": + if (!value.length) break + types.add(id) + value.forEach(({ fields }) => { + fields.forEach(({ type }) => { + collectTypesFromId(type) + }) + }) + break + case "tuple": + if (!value.length) break + types.add(id) + value.forEach(collectTypesFromId) + break + case "sequence": + types.add(id) + collectTypesFromId(value) + break + case "array": + types.add(id) + collectTypesFromId(value.type) + break + case "bitSequence": // bitSequence inner types are not stored + types.add(id) + // primitive and compact are not stored + } + } + + collectTypesFromId(metadata.extrinsic.call) + collectTypesFromId(metadata.extrinsic.address) + collectTypesFromId(metadata.extrinsic.signature) + metadata.extrinsic.signedExtensions.forEach(({ type, additionalSigned }) => { + collectTypesFromId(type) + collectTypesFromId(additionalSigned) + }) + + const sortedTypes = [...types].sort((a, b) => a - b) + return new Map(sortedTypes.map((value, idx) => [value, idx])) +} diff --git a/packages/merkleize-metadata/src/get-lookup.ts b/packages/merkleize-metadata/src/get-lookup.ts new file mode 100644 index 0000000..2881676 --- /dev/null +++ b/packages/merkleize-metadata/src/get-lookup.ts @@ -0,0 +1,119 @@ +import { Lookup, LookupValue, ScalePrimitive, TypeDef, TypeRef } from "./codecs" + +const bitSequenceBytes = { + u8: 1, + u16: 2, + u32: 4, + u64: 8, +} + +const constructTypeDef = ( + definitions: Map, + getTypeRef: (frameId: number) => TypeRef, + getPrimitive: (frameId: number) => ScalePrimitive["tag"] | null, + frameId: number, +): TypeDef[] => { + const { + def: { tag, value }, + } = definitions.get(frameId)! + switch (tag) { + case "composite": + return [ + { + tag, + value: value.map((f) => ({ + name: f.name, + typeName: f.typeName, + ty: getTypeRef(f.type), + })), + }, + ] + case "variant": { + return value.map((v) => ({ + tag: "enumeration", + value: { + name: v.name, + index: v.index, + fields: v.fields.map((f) => ({ + name: f.name, + typeName: f.typeName, + ty: getTypeRef(f.type), + })), + }, + })) + } + case "sequence": + return [ + { + tag, + value: getTypeRef(value), + }, + ] + case "array": + return [ + { + tag, + value: { + len: value.len, + typeParam: getTypeRef(value.type), + }, + }, + ] + case "tuple": + return [ + { + tag, + value: value.map(getTypeRef), + }, + ] + case "bitSequence": { + const primitive = getPrimitive(value.bitStoreType) + const numBytes = bitSequenceBytes[primitive as "u8"] + if (!numBytes) throw new Error("Invalid primitive for BitSequence") + + const storeOrderPath = definitions.get(value.bitOrderType)!.path + const leastSignificantBitFirst = storeOrderPath.includes("Lsb0") + if (!leastSignificantBitFirst && !storeOrderPath.includes("Msb0")) + throw new Error("BitOrderType not recognized") + + return [ + { + tag: "bitSequence", + value: { numBytes, leastSignificantBitFirst }, + }, + ] + } + } + + throw new Error(`FrameId(${frameId}) should have been filtered out`) +} + +export const getLookup = ( + definitions: Map, + accessibleTypes: Map, + getTypeRef: (frameId: number) => TypeRef, + getPrimitive: (frameId: number) => ScalePrimitive["tag"] | null, +) => { + const typeTree: Lookup = [] + ;[...accessibleTypes.entries()].forEach(([frameId, typeId]) => { + const { path } = definitions.get(frameId)! + constructTypeDef(definitions, getTypeRef, getPrimitive, frameId).forEach( + (typeDef) => { + typeTree.push({ + path, + typeId, + typeDef, + }) + }, + ) + }) + + typeTree.sort((a, b) => { + if (a.typeId !== b.typeId) return a.typeId - b.typeId // in general + // should only happen for variants + if (a.typeDef.tag !== "enumeration" || b.typeDef.tag !== "enumeration") + throw new Error("Found two types with same id") + return a.typeDef.value.index - b.typeDef.value.index + }) + return typeTree +} diff --git a/packages/merkleize-metadata/src/get-metadata.ts b/packages/merkleize-metadata/src/get-metadata.ts new file mode 100644 index 0000000..4ed8dee --- /dev/null +++ b/packages/merkleize-metadata/src/get-metadata.ts @@ -0,0 +1,18 @@ +import { + decAnyMetadata, + UnifiedMetadata, + type HexString, + unifyMetadata, +} from "@pezkuwi/bizinikiwi-bindings" + +export const getMetadata = ( + input: Uint8Array | HexString, +): UnifiedMetadata<15 | 16> => { + try { + const metadata = unifyMetadata(decAnyMetadata(input)) + if (metadata.version <= 14) throw new Error("Wrong metadata version") + return metadata as UnifiedMetadata<15 | 16> + } catch (e) { + throw e || new Error("Unable to decode metadata") + } +} diff --git a/packages/merkleize-metadata/src/main.ts b/packages/merkleize-metadata/src/main.ts new file mode 100644 index 0000000..ee91013 --- /dev/null +++ b/packages/merkleize-metadata/src/main.ts @@ -0,0 +1,294 @@ +import { + Blake3256, + HexString, + compact, + u32, +} from "@pezkuwi/bizinikiwi-bindings" +import { + ExtraInfo, + ExtrinsicMetadata, + LookupValue, + MetadataDigest, + ScalePrimitive, + TypeRef, + extraInfo, + extrinsicMetadata, + lookupType, + metadataDigest, + extrinsicDec, +} from "./codecs" +import { getAccessibleTypes } from "./get-accessible-types" +import { getLookup } from "./get-lookup" +import { getMetadata } from "./get-metadata" +import { compactTypeRefs, toBytes } from "./utils" +import { decodeAndCollectKnownLeafs } from "./decode-and-collect" +import { getProofData } from "./proof" +import { getDynamicBuilder, getLookupFn } from "@pezkuwi/metadata-builders" +import { mergeUint8 } from "@pezkuwi/papi-utils" + +export interface MetadataMerkleizer { + /** + * @returns Digest value of the metadata (aka its merkleized root-hash) + */ + digest: () => Uint8Array + + /** + * Get proof for an `Extrinsic`. + * + * @param transaction Encoded `Extrinsic`, both signed or unsigned. + * @param txAdditionalSigned Optionally collect types for given + * `additionalSigned` part of signedExtensions. + * @returns Encoded `Proof` + */ + getProofForExtrinsic: ( + transaction: Uint8Array | HexString, + txAdditionalSigned?: Uint8Array | HexString, + ) => Uint8Array + + /** + * Get proof for `ExtrinsicPayload` parts. + * + * @param callData Call data of the transaction. It has to be + * bare; i.e. + * without prepended compact length. + * @param includedInExtrinsic Extra part of the signed extensions, all of + * them concatenated. + * @param includedInSignedData Additional signed part of the signed + * extensions, all of them concatenated. + * @returns Encoded `Proof` + */ + getProofForExtrinsicParts: ( + callData: Uint8Array | HexString, + includedInExtrinsic: Uint8Array | HexString, + includedInSignedData: Uint8Array | HexString, + ) => Uint8Array + + /** + * Get proof for `ExtrinsicPayload`. + * + * @param extrinsicPayload Call data, extra part of signedExtensions and + * additional signed part of signedExtensions + * concatenated. It has to be bare; i.e. without + * prepended compact length. + * @returns Encoded `Proof` + */ + getProofForExtrinsicPayload: ( + extrinsicPayload: Uint8Array | HexString, + ) => Uint8Array +} + +const assertExpected = (name: string, expected: T, received?: T): void => { + if (received != null && received !== expected) + throw new Error( + `${name} not expected. Received ${received} expected ${expected}`, + ) +} + +export const merkleizeMetadata = ( + metadataBytes: Uint8Array | HexString, + { + decimals, + tokenSymbol, + ...hinted + }: { decimals: number; tokenSymbol: string } & Partial, +): MetadataMerkleizer => { + const metadata = getMetadata(metadataBytes) + + const checkedVersion = metadata.extrinsic.version.includes(4) ? 4 : null + if (checkedVersion == null) throw new Error("Only extrinsic v4 is supported") + + const { ss58Prefix, buildDefinition } = getDynamicBuilder( + getLookupFn(metadata), + ) + if (ss58Prefix == null) throw new Error("SS58 prefix not found in metadata") + assertExpected("SS58 prefix", ss58Prefix, hinted.base58Prefix) + const version = metadata.pallets + .find((x) => x.name === "System") + ?.constants.find((x) => x.name === "Version") + if (version == null) throw new Error("System.Version constant not found") + const { spec_name: specName, spec_version: specVersion } = buildDefinition( + version.type, + ).dec(version.value) + if (typeof specName !== "string" || typeof specVersion !== "number") + throw new Error("Spec name or spec version not found") + assertExpected("Spec name", specName, hinted.specName) + assertExpected("Spec version", specVersion, hinted.specVersion) + + const info: ExtraInfo = { + decimals, + tokenSymbol, + specVersion, + specName, + base58Prefix: ss58Prefix, + } + const definitions = new Map( + metadata.lookup.map((value) => [value.id, value]), + ) + const accessibleTypes = getAccessibleTypes(metadata, definitions) + const getPrimitive = (frameId: number): ScalePrimitive["tag"] | null => { + const { + def: { tag, value }, + } = definitions.get(frameId)! + + if (tag === "primitive") return value.tag + + if ((tag !== "composite" && tag !== "tuple") || value.length > 1) + throw new Error("The provided definition doesn't map to a primitive") + + return value.length === 0 + ? null // signals `void` + : getPrimitive(tag === "tuple" ? value[0] : value[0].type) + } + + const getTypeRef = (frameId: number): TypeRef => { + const { def } = definitions.get(frameId)! + + if (def.tag === "primitive") return { tag: def.value.tag, value: undefined } + + if (def.tag === "compact") { + const primitive = getPrimitive(def.value) + const tag = compactTypeRefs[primitive as "u8"] + if (!tag) throw new Error("Invalid primitive for Compact") + return { tag, value: undefined } + } + + return accessibleTypes.has(frameId) + ? { tag: "perId", value: accessibleTypes.get(frameId)! } + : { tag: "void", value: undefined } + } + + const extrinsic: ExtrinsicMetadata = { + version: checkedVersion, + addressTy: getTypeRef(metadata.extrinsic.address), + callTy: getTypeRef(metadata.extrinsic.call), + signatureTy: getTypeRef(metadata.extrinsic.signature), + signedExtensions: metadata.extrinsic.signedExtensions.map((se) => ({ + identifier: se.identifier, + includedInExtrinsic: getTypeRef(se.type), + includedInSignedData: getTypeRef(se.additionalSigned), + })), + } + + const lookup = getLookup( + definitions, + accessibleTypes, + getTypeRef, + getPrimitive, + ) + const lookupEncoded = lookup.map(lookupType.enc) + + let hashTree: Array | undefined + const getHashTree = (): Array => { + if (hashTree) return hashTree + + if (!lookupEncoded.length) return (hashTree = [new Uint8Array(32).fill(0)]) + + hashTree = new Array(lookupEncoded.length * 2 - 1) + + let leavesStartIdx = lookupEncoded.length - 1 + for (let i = 0; i < lookupEncoded.length; i++) + hashTree[leavesStartIdx + i] = Blake3256(lookupEncoded[i]) + + for (let i = hashTree.length - 2; i > 0; i -= 2) + hashTree[(i - 1) / 2] = Blake3256( + mergeUint8([hashTree[i], hashTree[i + 1]]), + ) + + return hashTree + } + + let digested: undefined | Uint8Array + const digest = () => { + if (digested) return digested + const rootLookupHash = getHashTree()[0] + + const digest: MetadataDigest = { + tag: "V1", + value: { + typeInformationTreeRoot: rootLookupHash, + extrinsicMetadataHash: Blake3256(extrinsicMetadata.enc(extrinsic)), + ...info, + }, + } + + return (digested = Blake3256(metadataDigest.enc(digest))) + } + + const generateProof = (knownIndexes: number[]) => { + const proofData = getProofData(lookupEncoded, knownIndexes) + + const hashTree = getHashTree() + const proofs = proofData.proofIdxs.map((idx) => hashTree[idx]) + + return mergeUint8([ + compact.enc(proofData.leaves.length), + ...proofData.leaves, + compact.enc(proofData.leafIdxs.length), + ...proofData.leafIdxs.map((x) => u32.enc(x)), + compact.enc(proofs.length), + ...proofs, + extrinsicMetadata.enc(extrinsic), + extraInfo.enc(info), + ]) + } + + const getProofForExtrinsicPayload = ( + extrinsicPayload: Uint8Array | HexString, + ) => { + const typeRefs: Array = [ + extrinsic.callTy, + ...extrinsic.signedExtensions.map((x) => x.includedInExtrinsic), + ...extrinsic.signedExtensions.map((x) => x.includedInSignedData), + ] + return generateProof( + decodeAndCollectKnownLeafs(extrinsicPayload, typeRefs, lookup), + ) + } + + const getProofForExtrinsicParts = ( + callData: Uint8Array | HexString, + includedInExtrinsic: Uint8Array | HexString, + includedInSignedData: Uint8Array | HexString, + ) => { + const bytes = mergeUint8( + [callData, includedInExtrinsic, includedInSignedData].map(toBytes), + ) + return getProofForExtrinsicPayload(bytes) + } + + const getProofForExtrinsic = ( + transaction: Uint8Array | HexString, + txAdditionalSigned?: Uint8Array | HexString, + ) => { + let [, { version, type }, bytes] = extrinsicDec(transaction) + + if (version !== extrinsic.version) + throw new Error("Incorrect extrinsic version") + + const typeRefs: TypeRef[] = + type === "signed" + ? [ + extrinsic.addressTy, + extrinsic.signatureTy, + ...extrinsic.signedExtensions.map((x) => x.includedInExtrinsic), + extrinsic.callTy, + ] + : [extrinsic.callTy] + + if (txAdditionalSigned) { + bytes = mergeUint8([bytes, toBytes(txAdditionalSigned)]) + typeRefs.push( + ...extrinsic.signedExtensions.map((x) => x.includedInSignedData), + ) + } + + return generateProof(decodeAndCollectKnownLeafs(bytes, typeRefs, lookup)) + } + + return { + digest, + getProofForExtrinsic, + getProofForExtrinsicParts, + getProofForExtrinsicPayload, + } +} diff --git a/packages/merkleize-metadata/src/proof.ts b/packages/merkleize-metadata/src/proof.ts new file mode 100644 index 0000000..b114933 --- /dev/null +++ b/packages/merkleize-metadata/src/proof.ts @@ -0,0 +1,57 @@ +const getLevelFromIdx = (idx: number) => Math.log2(idx + 1) | 0 +const getAncestorIdx = (from: number, nLevels: number) => + ((from + 1) >> nLevels) - 1 + +export function getProofData( + leaves: Array, + knownLeavesIdxs: Array, +) { + const knownLeaves = knownLeavesIdxs.map((idx) => leaves[idx]) + + const startingIdx = leaves.length - 1 + const leafIdxs = knownLeavesIdxs.map((idx) => startingIdx + idx) + + const proofIdxs: Array = [] + if (leafIdxs.length) { + const nLevels = getLevelFromIdx(leafIdxs.at(-1)!) + const splitPosition = Math.pow(2, nLevels) - 1 + const splitIdx = leafIdxs.findIndex((x) => x >= splitPosition) + if (splitIdx > 0) { + leafIdxs.unshift(...leafIdxs.splice(splitIdx)) + knownLeaves.unshift(...knownLeaves.splice(splitIdx)) + } + } + + let targetIdx = 0 + const traverse = (nodeIdx: number): void => { + if (targetIdx === leafIdxs.length) { + proofIdxs.push(nodeIdx) + return + } + + const target = leafIdxs[targetIdx] + if (target === nodeIdx) { + ++targetIdx + return + } + + const currentLevel = getLevelFromIdx(nodeIdx) + const targetLevel = getLevelFromIdx(target) + + if (nodeIdx !== getAncestorIdx(target, targetLevel - currentLevel)) { + proofIdxs.push(nodeIdx) + return + } + + const leftSon = 2 * nodeIdx + 1 + traverse(leftSon) + traverse(leftSon + 1) + } + traverse(0) + + return { + leaves: knownLeaves, + leafIdxs, + proofIdxs, + } +} diff --git a/packages/merkleize-metadata/src/utils.ts b/packages/merkleize-metadata/src/utils.ts new file mode 100644 index 0000000..d64498a --- /dev/null +++ b/packages/merkleize-metadata/src/utils.ts @@ -0,0 +1,15 @@ +import { type HexString } from "@pezkuwi/bizinikiwi-bindings" +import { fromHex } from "@pezkuwi/papi-utils" + +export const toBytes = (input: Uint8Array | HexString) => + typeof input === "string" ? fromHex(input) : input + +export const compactTypeRefs = { + null: "void" as const, + u8: "compactU8" as const, + u16: "compactU16" as const, + u32: "compactU32" as const, + u64: "compactU64" as const, + u128: "compactU128" as const, + u256: "compactU256" as const, +} diff --git a/packages/metadata-builders/README.md b/packages/metadata-builders/README.md index ca84905..691ba2d 100644 --- a/packages/metadata-builders/README.md +++ b/packages/metadata-builders/README.md @@ -1,4 +1,15 @@ -# @polkadot-api/metadata-builders +# @pezkuwi/metadata-builders + +**Source:** `src/` is [`@polkadot-api/metadata-builders`](https://www.npmjs.com/package/@polkadot-api/metadata-builders/v/0.13.2) 0.13.2 from +[polkadot-api](https://github.com/polkadot-api/polkadot-api) (MIT), with imports of +`@polkadot-api/*` renamed to `@pezkuwi/*` (`substrate-bindings` → `bizinikiwi-bindings`). `dist/` is its build; +`scripts/check-papi-sources.mjs` fails CI unless the source embedded in `dist/` equals `src/`. + +The upstream README follows. + +--- + +## @polkadot-api/metadata-builders This package has multiple functions that read a metadata object, denormalizes it, and builds other structures needed for different processes diff --git a/packages/metadata-builders/src/checksum-builder.ts b/packages/metadata-builders/src/checksum-builder.ts new file mode 100644 index 0000000..2d21085 --- /dev/null +++ b/packages/metadata-builders/src/checksum-builder.ts @@ -0,0 +1,530 @@ +import type { StringRecord } from "@pezkuwi/bizinikiwi-bindings" +import { h64 } from "@pezkuwi/bizinikiwi-bindings" +import { + LookupGraph, + buildLookupGraph, + getStronglyConnectedComponents, + getSubgraph, + mergeSCCsWithCommonNodes, +} from "./lookup-graph" +import { + ArrayVar, + LookupEntry, + MetadataLookup, + MetadataPrimitives, + StructVar, + TupleVar, + VoidVar, +} from "./lookups" + +const textEncoder = new TextEncoder() +const encodeText = textEncoder.encode.bind(textEncoder) + +const getChecksum = (values: Array) => { + const res = new Uint8Array(values.length * 8) + const dv = new DataView(res.buffer) + + for (let i = 0; i < values.length; i++) dv.setBigUint64(i * 8, values[i]) + + return h64(res) +} +const getStringChecksum = (values: Array) => + getChecksum(values.map((v) => h64(encodeText(v)))) + +type Shape = + | "primitive" + | "void" + | "vector" + | "tuple" + | "struct" + | "option" + | "result" + | "enum" +const shapeIds: Record = { + primitive: 0n, + vector: 1n, + tuple: 2n, + struct: 3n, + option: 4n, + result: 5n, + enum: 6n, + void: 7n, +} + +type RuntimePrimitives = + | "undefined" + | "number" + | "string" + | "bigint" + | "boolean" + | "bitSequence" + | "byteSequence" + | "accountId32" + | "accountId20" + +const runtimePrimitiveIds: Record = { + undefined: 0n, + number: 1n, + string: 2n, + bigint: 3n, + boolean: 4n, + bitSequence: 5n, // {bitsLen: number, bytes: Uint8Array} + byteSequence: 6n, // Binary + accountId32: 7n, // SS58String + accountId20: 8n, // EthAccount +} + +const metadataPrimitiveIds: Record = { + bool: runtimePrimitiveIds.boolean, + char: runtimePrimitiveIds.string, + str: runtimePrimitiveIds.string, + u8: runtimePrimitiveIds.number, + u16: runtimePrimitiveIds.number, + u32: runtimePrimitiveIds.number, + u64: runtimePrimitiveIds.bigint, + u128: runtimePrimitiveIds.bigint, + u256: runtimePrimitiveIds.bigint, + i8: runtimePrimitiveIds.number, + i16: runtimePrimitiveIds.number, + i32: runtimePrimitiveIds.number, + i64: runtimePrimitiveIds.bigint, + i128: runtimePrimitiveIds.bigint, + i256: runtimePrimitiveIds.bigint, +} + +const structLikeBuilder = ( + shapeId: bigint, + input: StringRecord, + innerChecksum: (value: T) => bigint, +) => { + const sortedEntries = Object.entries(input).sort(([a], [b]) => + a.localeCompare(b), + ) + const keysChecksum = getStringChecksum(sortedEntries.map(([key]) => key)) + const valuesChecksum = getChecksum( + sortedEntries.map(([, entry]) => innerChecksum(entry)), + ) + + return getChecksum([shapeId, keysChecksum, valuesChecksum]) +} + +const _buildChecksum = ( + input: LookupEntry, + buildNextChecksum: (entry: LookupEntry) => bigint, +): bigint => { + if (input.type === "primitive") + return getChecksum([shapeIds.primitive, metadataPrimitiveIds[input.value]]) + + if (input.type === "void") return getChecksum([shapeIds.void]) + + if (input.type === "compact") + return getChecksum([ + shapeIds.primitive, + runtimePrimitiveIds[input.isBig ? "bigint" : "number"], + ]) + + if (input.type === "bitSequence") + return getChecksum([shapeIds.primitive, runtimePrimitiveIds.bitSequence]) + + if (input.type === "AccountId32") { + return getChecksum([shapeIds.primitive, runtimePrimitiveIds.accountId32]) + } + + if (input.type === "AccountId20") { + return getChecksum([shapeIds.primitive, runtimePrimitiveIds.accountId20]) + } + + const buildVector = (entry: LookupEntry, length?: number) => { + const innerChecksum = buildNextChecksum(entry) + return getChecksum( + length !== undefined + ? [shapeIds.vector, innerChecksum, BigInt(length)] + : [shapeIds.vector, innerChecksum], + ) + } + + if (input.type === "array") { + const innerValue = input.value + if (innerValue.type === "primitive" && innerValue.value === "u8") { + return getChecksum([ + shapeIds.primitive, + runtimePrimitiveIds.byteSequence, + BigInt(input.len), + ]) + } + return buildVector(innerValue, input.len) + } + + if (input.type === "sequence") { + const innerValue = input.value + if (innerValue.type === "primitive" && innerValue.value === "u8") { + return getChecksum([shapeIds.primitive, runtimePrimitiveIds.byteSequence]) + } + return buildVector(innerValue) + } + + const buildTuple = (entries: LookupEntry[]) => + getChecksum([shapeIds.tuple, ...entries.map(buildNextChecksum)]) + + const buildStruct = (entries: StringRecord) => + structLikeBuilder(shapeIds.struct, entries, buildNextChecksum) + + if (input.type === "tuple") return buildTuple(input.value) + + if (input.type === "struct") return buildStruct(input.value) + + if (input.type === "option") + return getChecksum([shapeIds.option, buildNextChecksum(input.value)]) + + if (input.type === "result") + return getChecksum([ + shapeIds.result, + buildNextChecksum(input.value.ok), + buildNextChecksum(input.value.ko), + ]) + + return structLikeBuilder(shapeIds.enum, input.value, (entry) => { + if (entry.type === "lookupEntry") return buildNextChecksum(entry.value) + switch (entry.type) { + case "void": + return getChecksum([shapeIds.void]) + case "tuple": + return buildTuple(entry.value) + case "struct": + return buildStruct(entry.value) + case "array": + return buildVector(entry.value, entry.len) + } + }) +} + +const sortCyclicGroups = (groups: Array>, graph: LookupGraph) => { + const getReachableNodes = (group: Set) => { + const result = new Set() + const toVisit = Array.from(group) + while (toVisit.length) { + const id = toVisit.pop()! + if (result.has(id)) continue + result.add(id) + + graph.get(id)?.refs.forEach((id) => toVisit.push(id)) + } + + return Array.from(result) + } + + const result: Array> = new Array() + + function dependentsFirst(group: Set) { + if (result.includes(group)) return + const dependents = groups.filter( + (candidate) => + candidate !== group && + getReachableNodes(group).some((node) => candidate.has(node)), + ) + dependents.forEach((group) => dependentsFirst(group)) + if (result.includes(group)) return + result.push(group) + } + + groups.forEach((group) => dependentsFirst(group)) + return result +} + +function iterateChecksums( + group: Set, + iterations: number, + cache: Map, + graph: LookupGraph, +) { + // Keep the values that are getting changed on each iteration in a separate + // cache, because two nodes referencing the same one should read the same + // previous iteration checksum for that node. + const groupReadCache = new Map([...group].map((id) => [id, 0n])) + const groupWriteCache = new Map() + + const recursiveBuildChecksum = ( + entry: LookupEntry, + // The first call has to skip the cache, otherwise it would return the + // previous iteration result. + skipCache = true, + ): bigint => { + if (!skipCache && (groupReadCache.has(entry.id) || cache.has(entry.id))) { + return groupReadCache.get(entry.id) ?? cache.get(entry.id)! + } + const result = _buildChecksum(entry, (nextEntry) => + recursiveBuildChecksum(nextEntry, false), + ) + if (group.has(entry.id)) { + groupWriteCache.set(entry.id, result) + } else { + cache.set(entry.id, result) + } + return result + } + + for (let i = 0; i < iterations; i++) { + group.forEach((id) => recursiveBuildChecksum(graph.get(id)!.entry)) + + group.forEach((id) => groupReadCache.set(id, groupWriteCache.get(id)!)) + } + + return groupReadCache +} + +function getMirroredNodes( + cyclicGroups: Array>, + graph: LookupGraph, +) { + const maxSize = cyclicGroups.reduce( + (acc, group) => Math.max(acc, group.size), + 0, + ) + const allEntries = new Set([...graph.values()].map((v) => v.entry.id)) + + const resultingChecksums = iterateChecksums( + allEntries, + maxSize, + // Cache won't be used, since it's using the internal one for every node. + new Map(), + graph, + ) + + const checksumToNodes = new Map() + for (const id of allEntries) { + const checksum = resultingChecksums.get(id) + if (checksum == undefined) throw new Error("Unreachable") + if (!checksumToNodes.has(checksum)) { + checksumToNodes.set(checksum, []) + } + checksumToNodes.get(checksum)!.push(id) + } + + const checksumsWithDuplicates = [...checksumToNodes.entries()].filter( + ([, nodes]) => nodes.length > 1, + ) + + const duplicatesMap: Record = {} + checksumsWithDuplicates.forEach(([, nodes]) => { + nodes.forEach((n) => (duplicatesMap[n] = nodes)) + }) + + return duplicatesMap +} + +const buildChecksum = ( + entry: LookupEntry, + cache: Map, + graph: LookupGraph, +) => { + if (cache.has(entry.id)) return cache.get(entry.id)! + + const subGraph = getSubgraph(entry.id, graph) + + const cycles = getStronglyConnectedComponents(subGraph).filter( + // SCCs can be of length=1, but for those we're only interested with those that are circular with themselves + (group) => group.size > 1 || isSelfCircular(group, subGraph), + ) + const cyclicGroups = mergeSCCsWithCommonNodes(cycles).filter((group) => { + // Exclude groups that were previously calculated + return !cache.has(group.values().next().value!) + }) + const mirrored = getMirroredNodes(cyclicGroups, subGraph) + const sortedCyclicGroups = sortCyclicGroups(cyclicGroups, subGraph) + + sortedCyclicGroups.forEach((group) => { + if (cache.has(group.values().next().value!)) { + // exclude mirrored groups + return + } + + const result = iterateChecksums(group, group.size, cache, graph) + group.forEach((id) => { + const checksum = result.get(id)! + if (id in mirrored) { + mirrored[id].forEach((id) => cache.set(id, checksum)) + } else { + cache.set(id, checksum) + } + }) + }) + + const getChecksum = (entry: LookupEntry) => { + if (cache.has(entry.id)) return cache.get(entry.id)! + return _buildChecksum(entry, getChecksum) + } + + return getChecksum(entry) +} + +const isSelfCircular = (group: Set, graph: LookupGraph) => { + if (group.size !== 1) return false + const [id] = group + + return graph.get(id)!.refs.has(id) +} + +export const getChecksumBuilder = (getLookupEntryDef: MetadataLookup) => { + const { metadata } = getLookupEntryDef + const graph = buildLookupGraph(getLookupEntryDef, metadata.lookup.length) + + const cache = new Map() + + const buildDefinition = (id: number): bigint => + buildChecksum(getLookupEntryDef(id), cache, graph) + + const buildStorage = (pallet: string, entry: string): bigint | null => { + try { + const storageEntry = metadata.pallets + .find((x) => x.name === pallet)! + .storage!.items.find((s) => s.name === entry)! + + if (storageEntry.type.tag === "plain") + return buildDefinition(storageEntry.type.value) + + const { key, value } = storageEntry.type.value + const val = buildDefinition(value) + const returnKey = buildDefinition(key) + return getChecksum([val, returnKey]) + } catch (_) { + return null + } + } + + const buildViewFns = (pallet: string, entry: string): bigint | null => { + try { + const viewFn = metadata.pallets + .find((x) => x.name === pallet) + ?.viewFns.find((x) => x.name === entry) + if (!viewFn) throw null + + const argNamesChecksum = getStringChecksum( + viewFn.inputs.map((x) => x.name), + ) + const argValuesChecksum = getChecksum( + viewFn.inputs.map((x) => buildDefinition(x.type)), + ) + const outputChecksum = buildDefinition(viewFn.output) + + return getChecksum([argNamesChecksum, argValuesChecksum, outputChecksum]) + } catch (_) { + return null + } + } + + const buildRuntimeCall = (api: string, method: string): bigint | null => { + try { + const entry = metadata.apis + .find((x) => x.name === api) + ?.methods.find((x) => x.name === method) + if (!entry) throw null + + const argNamesChecksum = getStringChecksum( + entry.inputs.map((x) => x.name), + ) + const argValuesChecksum = getChecksum( + entry.inputs.map((x) => buildDefinition(x.type)), + ) + const outputChecksum = buildDefinition(entry.output) + + return getChecksum([argNamesChecksum, argValuesChecksum, outputChecksum]) + } catch (_) { + return null + } + } + + const buildComposite = ( + input: TupleVar | StructVar | VoidVar | ArrayVar, + ): bigint => { + if (input.type === "void") return getChecksum([0n]) + + if (input.type === "tuple") { + const values = Object.values(input.value).map((entry) => + buildDefinition(entry.id), + ) + + return getChecksum([shapeIds.tuple, ...values]) + } + + if (input.type === "array") { + return getChecksum([ + shapeIds.vector, + buildDefinition(input.value.id), + BigInt(input.len), + ]) + } + + // Otherwise struct + return structLikeBuilder(shapeIds.struct, input.value, (entry) => + buildDefinition(entry.id), + ) + } + + const buildNamedTuple = (input: StructVar): bigint => { + return structLikeBuilder(shapeIds.tuple, input.value, (entry) => + buildDefinition(entry.id), + ) + } + + const variantShapeId = { + errors: 1n, + events: 2n, + calls: 3n, + } + const buildVariant = + (variantType: "errors" | "events" | "calls") => + (pallet: string, name: string): bigint | null => { + try { + const palletEntry = metadata.pallets.find((x) => x.name === pallet)! + const enumLookup = getLookupEntryDef(palletEntry[variantType]!.type) + buildDefinition(enumLookup.id) + + if (enumLookup.type !== "enum") throw null + const entry = enumLookup.value[name] + const valueChecksum = + entry.type === "lookupEntry" + ? buildDefinition(entry.value.id) + : buildComposite(entry) + return getChecksum([variantShapeId[variantType], valueChecksum]) + } catch (_) { + return null + } + } + + const buildConstant = ( + pallet: string, + constantName: string, + ): bigint | null => { + try { + const storageEntry = metadata.pallets + .find((x) => x.name === pallet)! + .constants!.find((s) => s.name === constantName)! + + return buildDefinition(storageEntry.type) + } catch (_) { + return null + } + } + + const toStringEnhancer = + >( + fn: (...args: Args) => bigint | null, + ): ((...args: Args) => string | null) => + (...args) => + fn(...args)?.toString(32) ?? null + + return { + buildDefinition: toStringEnhancer(buildDefinition), + buildRuntimeCall: toStringEnhancer(buildRuntimeCall), + buildStorage: toStringEnhancer(buildStorage), + buildViewFns: toStringEnhancer(buildViewFns), + buildCall: toStringEnhancer(buildVariant("calls")), + buildEvent: toStringEnhancer(buildVariant("events")), + buildError: toStringEnhancer(buildVariant("errors")), + buildConstant: toStringEnhancer(buildConstant), + buildComposite: toStringEnhancer(buildComposite), + buildNamedTuple: toStringEnhancer(buildNamedTuple), + getAllGeneratedChecksums: () => + Array.from(cache.values()).map((v) => v.toString(32)), + } +} diff --git a/packages/metadata-builders/src/dynamic-builder.ts b/packages/metadata-builders/src/dynamic-builder.ts new file mode 100644 index 0000000..de933e8 --- /dev/null +++ b/packages/metadata-builders/src/dynamic-builder.ts @@ -0,0 +1,191 @@ +import type { Codec, StringRecord } from "@pezkuwi/bizinikiwi-bindings" +import * as scale from "@pezkuwi/bizinikiwi-bindings" +import { mapObject } from "@pezkuwi/papi-utils" +import type { EnumVar, MetadataLookup } from "./lookups" +import { getLookupCodecBuilder } from "./lookup-codec-builder" + +const nullCodec = scale.enhanceCodec( + scale._void, + () => undefined, + () => null, +) + +export const getDynamicBuilder = (getLookupEntryDef: MetadataLookup) => { + const { metadata } = getLookupEntryDef + let buildDefinition = getLookupCodecBuilder(getLookupEntryDef) + + const prefix = metadata.pallets + .find((x) => x.name === "System") + ?.constants.find((x) => x.name === "SS58Prefix") + + let ss58Prefix: number | undefined + if (prefix) { + try { + const prefixVal = buildDefinition(prefix.type).dec(prefix.value) + if (typeof prefixVal === "number") { + ss58Prefix = prefixVal + buildDefinition = getLookupCodecBuilder( + getLookupEntryDef, + scale.AccountId(prefixVal), + ) + } + } catch (_) {} + } + + const storagePallets = new Map>() + + const buildStorage = (pallet: string, entry: string) => { + let storagePallet = storagePallets.get(pallet) + if (!storagePallet) + storagePallets.set(pallet, (storagePallet = scale.Storage(pallet))) + + const storageEntry = metadata.pallets + .find((x) => x.name === pallet)! + .storage!.items.find((s) => s.name === entry)! + + // if val is `void` it decodes to `undefined`, making it impossible + // to differentiate from a non-existant key + // therefore, if the key exists => null, if it doesn't => undefined + const withNullVoid = (codec: Codec) => + codec === scale._void ? nullCodec : codec + + const storageWithFallback = ( + len: number, + value: Codec, + ...args: Parameters> + ) => { + const keys = storagePallet!(...args) + const [, ...encodersWithHash] = args + return { + args: scale.Tuple(...encodersWithHash.map(([codec]) => codec)), + keys, + value, + len, + fallback: + storageEntry.modifier === 1 + ? value.dec(storageEntry.fallback) + : undefined, + } + } + + if (storageEntry.type.tag === "plain") + return storageWithFallback( + 0, + withNullVoid(buildDefinition(storageEntry.type.value)), + entry, + ) + + const { key, value, hashers } = storageEntry.type.value + const val = withNullVoid(buildDefinition(value)) + const hashes = hashers.map((x) => scale[x.tag]) + + const hashArgs: scale.EncoderWithHash[] = (() => { + if (hashes.length === 1) { + return [[buildDefinition(key), hashes[0]]] + } + + const keyDef = getLookupEntryDef(key) + + switch (keyDef.type) { + case "array": + return hashes.map((hash) => [buildDefinition(keyDef.value.id), hash]) + case "tuple": + return keyDef.value.map((x, idx) => [ + buildDefinition(x.id), + hashes[idx], + ]) + default: + throw new Error("Invalid key type") + } + })() + + return storageWithFallback(hashes.length, val, entry, ...hashArgs) + } + + const buildEnumEntry = ( + entry: EnumVar["value"][keyof EnumVar["value"]], + ): Codec => { + switch (entry.type) { + case "void": + return scale._void + case "lookupEntry": + return buildDefinition(entry.value.id) + case "tuple": + return scale.Tuple( + ...Object.values(entry.value).map((l) => buildDefinition(l.id)), + ) + case "struct": + return scale.Struct( + mapObject(entry.value, (x) => buildDefinition(x.id)) as StringRecord< + Codec + >, + ) + case "array": + return scale.Vector(buildDefinition(entry.value.id), entry.len) + } + } + + const buildConstant = (pallet: string, constantName: string) => { + const storageEntry = metadata.pallets + .find((x) => x.name === pallet)! + .constants!.find((s) => s.name === constantName)! + + return buildDefinition(storageEntry.type as number) + } + + const buildVariant = + (type: "errors" | "events" | "calls") => + ( + pallet: string, + name: string, + ): { + codec: Codec + location: [number, number] + } => { + const palletEntry = metadata.pallets.find((x) => x.name === pallet)! + const lookup = getLookupEntryDef(palletEntry[type]!.type) + if (lookup.type !== "enum") throw null + const entry = lookup.value[name] + + return { + location: [palletEntry.index, entry.idx], + codec: buildEnumEntry(lookup.value[name]), + } + } + + const buildViewFn = (pallet: string, entry: string) => { + const fn = metadata.pallets + .find((x) => x.name === pallet) + ?.viewFns.find((x) => x.name === entry) + if (!fn) throw null + + return { + args: scale.Tuple(...fn.inputs.map((x) => buildDefinition(x.type))), + value: buildDefinition(fn.output), + } + } + + const buildRuntimeCall = (api: string, method: string) => { + const entry = metadata.apis + .find((x) => x.name === api) + ?.methods.find((x) => x.name === method) + if (!entry) throw null + + return { + args: scale.Tuple(...entry.inputs.map((x) => buildDefinition(x.type))), + value: buildDefinition(entry.output), + } + } + + return { + buildDefinition, + buildStorage, + buildEvent: buildVariant("events"), + buildError: buildVariant("errors"), + buildViewFn, + buildRuntimeCall, + buildCall: buildVariant("calls"), + buildConstant, + ss58Prefix, + } +} diff --git a/packages/metadata-builders/src/lookup-codec-builder.ts b/packages/metadata-builders/src/lookup-codec-builder.ts new file mode 100644 index 0000000..9a99c55 --- /dev/null +++ b/packages/metadata-builders/src/lookup-codec-builder.ts @@ -0,0 +1,119 @@ +import type { Codec, StringRecord } from "@pezkuwi/bizinikiwi-bindings" +import * as scale from "@pezkuwi/bizinikiwi-bindings" +import type { LookupEntry } from "./lookups" +import { withCache } from "./with-cache" + +const _bytes = scale.Bin() + +const _buildCodec = ( + input: LookupEntry, + cache: Map>, + stack: Set, + _accountId: Codec, +): Codec => { + if (input.type === "primitive") return scale[input.value] + if (input.type === "void") return scale._void + if (input.type === "AccountId32") return _accountId + if (input.type === "AccountId20") return scale.ethAccount + if (input.type === "compact") + return input.isBig ? scale.compactBn : scale.compactNumber + if (input.type === "bitSequence") return scale.BitSeq(input.isLSB) + + const buildNextCodec = (nextInput: LookupEntry): Codec => + buildCodec(nextInput, cache, stack, _accountId) + + const buildVector = (inner: LookupEntry, len?: number) => { + const innerCodec = buildNextCodec(inner) + return len ? scale.Vector(innerCodec, len) : scale.Vector(innerCodec) + } + + const buildTuple = (value: LookupEntry[]) => + scale.Tuple(...value.map(buildNextCodec)) + + const buildStruct = (value: StringRecord) => { + const inner = Object.fromEntries( + Object.entries(value).map(([key, value]) => [key, buildNextCodec(value)]), + ) as StringRecord> + return scale.Struct(inner) + } + + if ( + input.type === "sequence" && + input.value.type === "primitive" && + input.value.value === "u8" + ) { + return _bytes + } + + if (input.type === "array") { + // Bytes case + if (input.value.type === "primitive" && input.value.value === "u8") + return scale.Bin(input.len) + + return buildVector(input.value, input.len) + } + + if (input.type === "sequence") return buildVector(input.value) + if (input.type === "tuple") return buildTuple(input.value) + if (input.type === "struct") return buildStruct(input.value) + + if (input.type === "option") return scale.Option(buildNextCodec(input.value)) + + if (input.type === "result") + return scale.Result( + buildNextCodec(input.value.ok), + buildNextCodec(input.value.ko), + ) + + // it has to be an enum by now + const dependencies = Object.values(input.value).map((v) => { + switch (v.type) { + case "void": + return scale._void + case "lookupEntry": + return buildNextCodec(v.value) + case "tuple": + return buildTuple(v.value) + case "struct": + return buildStruct(v.value) + case "array": + return buildVector(v.value, v.len) + } + }) + + const inner = Object.fromEntries( + Object.keys(input.value).map((key, idx) => { + return [key, dependencies[idx]] + }), + ) as StringRecord> + + const indexes = Object.values(input.value).map((x) => x.idx) + const areIndexesSorted = indexes.every((idx, i) => idx === i) + + const variantCodec = areIndexesSorted + ? scale.Variant(inner) + : scale.Variant(inner, indexes as any) + return input.byteLength + ? fixedSizeCodec(variantCodec, input.byteLength) + : variantCodec +} +const buildCodec = withCache(_buildCodec, scale.Self, (res) => res) + +export const getLookupCodecBuilder = ( + lookup: (id: number) => LookupEntry, + accountId = scale.AccountId(), +) => { + const cache = new Map() + const buildDefinition = (id: number) => + buildCodec(lookup(id), cache, new Set(), accountId) + + return (id: number) => buildDefinition(id) +} + +const fixedSizeCodec = (codec: Codec, size: number): Codec => { + const allBytes = scale.Bytes(size) + return scale.createCodec( + (value: T) => allBytes.enc(codec.enc(value)), + (data) => codec.dec(allBytes.dec(data)), + ) +} diff --git a/packages/metadata-builders/src/lookup-graph.ts b/packages/metadata-builders/src/lookup-graph.ts new file mode 100644 index 0000000..33ce5bd --- /dev/null +++ b/packages/metadata-builders/src/lookup-graph.ts @@ -0,0 +1,214 @@ +import { LookupEntry } from "./lookups" + +export type LookupGraph = Map< + number, + { entry: LookupEntry; backRefs: Set; refs: Set } +> + +export function buildLookupGraph( + lookupFn: (id: number) => LookupEntry, + lookupLength: number, +): LookupGraph { + const result: LookupGraph = new Map() + const visited = new Set() + + const addEdge = (from: number, to: number) => { + if (!result.has(from)) + result.set(from, { + entry: lookupFn(from), + backRefs: new Set(), + refs: new Set(), + }) + if (!result.has(to)) + result.set(to, { + entry: lookupFn(to), + backRefs: new Set(), + refs: new Set(), + }) + result.get(from)!.refs.add(to) + result.get(to)!.backRefs.add(from) + } + + for (let i = 0; i < lookupLength; i++) { + const entry = lookupFn(i) + if (i !== entry.id) { + // Lookup solved a pointer, but we still need to add that node into our + // graph in case someone access it from there. + addEdge(i, entry.id) + } + if (visited.has(entry.id)) continue + visited.add(entry.id) + + switch (entry.type) { + case "array": + case "option": + case "sequence": + addEdge(entry.id, entry.value.id) + break + case "enum": + Object.values(entry.value).forEach((enumEntry) => { + switch (enumEntry.type) { + case "array": + case "lookupEntry": + addEdge(entry.id, enumEntry.value.id) + break + case "struct": + case "tuple": + Object.values(enumEntry.value).forEach((v) => + addEdge(entry.id, v.id), + ) + break + } + }) + break + case "result": + addEdge(entry.id, entry.value.ok.id) + addEdge(entry.id, entry.value.ko.id) + break + case "struct": + case "tuple": + Object.values(entry.value).forEach((v) => addEdge(entry.id, v.id)) + break + } + + // It could be that this node is not being referenced by any other type + // nor it references anything. We still have to add it into the graph. + if (!result.has(entry.id)) { + result.set(entry.id, { + backRefs: new Set(), + refs: new Set(), + entry, + }) + } + } + + return result +} + +const subgraphCache = new WeakMap>() +function _getSubgraph( + id: number, + graph: LookupGraph, + result: LookupGraph, + cache: Map, +) { + if (result.has(id)) return + const node = graph.get(id)! + result.set(id, node) + cache.set(id, result) + + node.refs.forEach((ref) => _getSubgraph(ref, graph, result, cache)) + node.backRefs.forEach((ref) => _getSubgraph(ref, graph, result, cache)) +} + +export function getSubgraph(id: number, graph: LookupGraph) { + if (!subgraphCache.has(graph)) { + subgraphCache.set(graph, new Map()) + } + const cache = subgraphCache.get(graph)! + if (cache.has(id)) return cache.get(id)! + + const result: LookupGraph = new Map() + _getSubgraph(id, graph, result, cache) + return result +} + +export function getStronglyConnectedComponents(graph: LookupGraph) { + // Tarjan's strongly connected components + const tarjanState = new Map< + number, + { + index: number + lowLink: number + onStack: boolean + } + >() + let index = 0 + const stack: number[] = [] + const result: Array> = [] + + function strongConnect(v: number): void { + const state = { + index: index, + lowLink: index, + onStack: true, + } + tarjanState.set(v, state) + index++ + stack.push(v) + + const edges = graph.get(v)!.refs + for (let w of edges) { + const edgeState = tarjanState.get(w) + if (!edgeState) { + strongConnect(w) + state.lowLink = Math.min(state.lowLink, tarjanState.get(w)!.lowLink) + } else if (edgeState.onStack) { + state.lowLink = Math.min(state.lowLink, edgeState.index) + } + } + + if (state.lowLink === state.index) { + const component = new Set() + + let poppedNode = -1 + do { + poppedNode = stack.pop()! + tarjanState.get(poppedNode)!.onStack = false + component.add(poppedNode) + } while (poppedNode !== v) + + result.push(component) + } + } + + for (const node of graph.keys()) { + if (!tarjanState.has(node)) { + strongConnect(node) + } + } + + return result +} + +export function mergeSCCsWithCommonNodes( + stronglyConnectedComponents: Array>, +) { + /** + * For Nodes that are shared between two sets of SCCs, we need to calculate + * the checksum for the both of them, which wouldn't work (it would give + * different checksums). + * So we merge the SCCs that are using shared nodes into one group. + */ + const scc = stronglyConnectedComponents + const ungroupedCycles = new Set(scc.map((_, i) => i)) + const edges = new Map(scc.map((_, i) => [i, new Set()])) + scc.forEach((cycle, i) => { + scc.slice(i + 1).forEach((otherCycle, _j) => { + const j = _j + i + 1 + const combined = new Set([...cycle, ...otherCycle]) + if (combined.size !== cycle.size + otherCycle.size) { + edges.get(i)!.add(j) + edges.get(j)!.add(i) + } + }) + }) + const groups: Array> = [] + + while (ungroupedCycles.size) { + const group = new Set() + const toVisit = [ungroupedCycles.values().next().value!] + while (toVisit.length) { + const idx = toVisit.pop()! + if (!ungroupedCycles.has(idx)) continue + ungroupedCycles.delete(idx) + + const cycle = scc[idx] + cycle.forEach((v) => group.add(Number(v))) + edges.get(idx)!.forEach((n) => toVisit.push(n)) + } + groups.push(group) + } + + return groups +} diff --git a/packages/metadata-builders/src/lookups.ts b/packages/metadata-builders/src/lookups.ts new file mode 100644 index 0000000..28b1030 --- /dev/null +++ b/packages/metadata-builders/src/lookups.ts @@ -0,0 +1,456 @@ +import type { + StringRecord, + UnifiedMetadata, + V14Lookup, +} from "@pezkuwi/bizinikiwi-bindings" + +export type SignedPrimitive = "i8" | "i16" | "i32" | "i64" | "i128" | "i256" +export type UnsignedPrimitive = "u8" | "u16" | "u32" | "u64" | "u128" | "u256" +export type MetadataPrimitives = + | "bool" + | "char" + | "str" + | SignedPrimitive + | UnsignedPrimitive + +export type PrimitiveVar = { + type: "primitive" + value: MetadataPrimitives +} + +export type VoidVar = { type: "void" } +export type CompactVar = { + type: "compact" + isBig: boolean + size: UnsignedPrimitive +} +export type BitSequenceVar = { + type: "bitSequence" + isLSB: boolean +} +export type AccountId32 = { type: "AccountId32" } +export type AccountId20 = { type: "AccountId20" } +export type TerminalVar = + | PrimitiveVar + | VoidVar + | CompactVar + | BitSequenceVar + | AccountId32 + | AccountId20 + +/* Array-like vars: + * - TupleVar: Mixed types, fixed length + * - Sequence: One type, arbitrary length + * - Array: One type, fixed length + */ +export type TupleVar = { + type: "tuple" + value: LookupEntry[] + innerDocs: Array +} +export type StructVar = { + type: "struct" + value: StringRecord + innerDocs: StringRecord +} +export type EnumVar = { + type: "enum" + value: StringRecord< + ( + | { type: "lookupEntry"; value: LookupEntry } + | VoidVar + | TupleVar + | StructVar + | ArrayVar + ) & { idx: number } + > + innerDocs: StringRecord + byteLength?: number +} +export type OptionVar = { + type: "option" + value: LookupEntry +} +export type ResultVar = { + type: "result" + value: { ok: LookupEntry; ko: LookupEntry } +} +export type SequenceVar = { + type: "sequence" + value: LookupEntry +} +export type ArrayVar = { + type: "array" + value: LookupEntry + len: number +} + +export type ComposedVar = + | TupleVar + | StructVar + | SequenceVar + | ArrayVar + | OptionVar + | ResultVar + | EnumVar + +export type Var = TerminalVar | ComposedVar + +export type LookupEntry = { + id: number +} & Var + +const isBytes = (value: LookupEntry, nBytes: number) => + value.type === "array" && + value.len === nBytes && + value.value.type === "primitive" && + value.value.value === "u8" + +const _void: VoidVar = { type: "void" } + +export interface MetadataLookup { + (id: number): LookupEntry + metadata: UnifiedMetadata + call: number | null +} + +const _denormalizeLookup = ( + lookupData: V14Lookup, + customMap: (value: V14Lookup[number]) => Var | null = () => null, +): ((id: number) => LookupEntry) => { + const lookups = new Map() + const from = new Set() + + const withCache = ( + fn: (id: number) => Var, + ): ((id: number) => LookupEntry) => { + return (id) => { + let entry = lookups.get(id) + + if (entry) return entry + + if (from.has(id)) { + const entry = { + id, + } as LookupEntry + + lookups.set(id, entry) + return entry + } + + from.add(id) + const value = fn(id) + entry = lookups.get(id) + + if (entry) { + Object.assign(entry, value) + } else { + entry = { + id, + ...value, + } + lookups.set(id, entry!) + } + from.delete(id) + return entry + } + } + + let isAccountId32SearchOn = true + let isAccountId20SearchOn = true + const getLookupEntryDef = withCache((id): Var => { + const custom = customMap(lookupData[id]) + if (custom) return custom + + const { def, path, params } = lookupData[id] + + if (def.tag === "composite") { + if (def.value.length === 0) return _void + + // used to be a "pointer" + if (def.value.length === 1) { + const inner = getLookupEntryDef(def.value[0].type as number) + + if ( + isAccountId32SearchOn && + path.at(-1) === "AccountId32" && + isBytes(inner, 32) + ) { + isAccountId32SearchOn = false + return { type: "AccountId32" } + } + + if ( + isAccountId20SearchOn && + path.at(-1) === "AccountId20" && + isBytes(inner, 20) + ) { + isAccountId20SearchOn = false + return { type: "AccountId20" } + } + + return inner + } + + return getComplexVar(def.value) + } + + if (def.tag === "variant") { + if ( + path.length === 1 && + path[0] === "Option" && + params.length === 1 && + params[0].name === "T" + ) { + const value = getLookupEntryDef(params[0].type!) + return value.type === "void" + ? // Option would return a Codec which makes no sense + // Therefore, we better treat it as a bool + { type: "primitive", value: "bool" } + : { + type: "option", + value, + } + } + + if ( + path.length === 1 && + path[0] === "Result" && + params.length === 2 && + params[0].name === "T" && + params[1].name === "E" + ) { + return { + type: "result", + value: { + ok: getLookupEntryDef(params[0].type as number), + ko: getLookupEntryDef(params[1].type as number), + }, + } + } + if (def.value.length === 0) return _void + + const enumValue: StringRecord = + {} + const enumDocs: StringRecord = {} + + def.value.forEach((x) => { + const key = x.name + enumDocs[key] = x.docs + + if (x.fields.length === 0) { + enumValue[key] = { ..._void, idx: x.index } + return + } + + if (x.fields.length === 1 && !x.fields[0].name) { + enumValue[key] = { + type: "lookupEntry", + value: getLookupEntryDef(x.fields[0].type), + idx: x.index, + } + return + } + + enumValue[key] = { ...getComplexVar(x.fields), idx: x.index } + }) + + return { + type: "enum", + value: enumValue, + innerDocs: enumDocs, + } + } + + if (def.tag === "sequence") + return { + type: "sequence", + value: getLookupEntryDef(def.value as number), + } + + if (def.tag === "array") { + const { len } = def.value + const value = getLookupEntryDef(def.value.type) + + return !len || value.type === "void" + ? _void + : len > 1 + ? { + type: "array", + value, + len: def.value.len, + } + : value + } + + if (def.tag === "tuple") { + if (def.value.length === 0) return _void + + return def.value.length > 1 + ? getArrayOrTuple( + def.value.map((x) => getLookupEntryDef(x as number)), + def.value.map((x) => lookupData[x].docs), + ) + : getLookupEntryDef(def.value[0] as number) // use to be a "pointer" + } + + if (def.tag === "primitive") { + return { + type: "primitive", + value: def.value.tag, + } + } + + if (def.tag === "compact") { + const translated = getLookupEntryDef(def.value) as PrimitiveVar | VoidVar + if (translated.type === "void") return _void + + const isBig = Number(translated.value.slice(1)) > 32 + + return { + type: "compact", + isBig, + size: translated.value as UnsignedPrimitive, + } + } + + // bitSequence + return { + type: def.tag, + isLSB: (lookupData[def.value.bitOrderType].path.at(-1) ?? "LSB") + .toUpperCase() + .startsWith("LSB"), + } + }) + + const getComplexVar = ( + input: Array<{ type: number; name?: string; docs: string[] }>, + ): TupleVar | StructVar | ArrayVar | VoidVar => { + let allKey = true + + const values: Record = {} + const innerDocs: Record = {} + + input.forEach((x, idx) => { + allKey = allKey && !!x.name + const key = x.name || idx + const value = getLookupEntryDef(x.type as number) + if (value.type !== "void") { + values[key] = value + innerDocs[key] = x.docs + } + }) + return allKey + ? { + type: "struct", + value: values as StringRecord, + innerDocs: innerDocs as StringRecord, + } + : getArrayOrTuple(Object.values(values), Object.values(innerDocs)) + } + + const getArrayOrTuple = ( + values: Array, + innerDocs: Array, + ): TupleVar | ArrayVar | VoidVar => { + if ( + values.every((v) => v.id === values[0].id) && + innerDocs.every((doc) => !doc.length) + ) { + const [value] = values + return value.type === "void" + ? _void + : { + type: "array", + value: values[0], + len: values.length, + } + } + return { + type: "tuple", + value: values, + innerDocs: innerDocs, + } + } + + return getLookupEntryDef +} + +export const denormalizeLookup = (lookupData: V14Lookup) => + _denormalizeLookup(lookupData) + +export const getLookupFn = (metadata: UnifiedMetadata): MetadataLookup => { + const getLookupEntryDef = _denormalizeLookup(metadata.lookup, ({ def }) => { + if (def.tag === "composite") { + const moduleErrorLength = getModuleErrorLength(def) + if (moduleErrorLength) { + return { + type: "enum", + innerDocs: {}, + value: Object.fromEntries( + metadata.pallets.map((p) => [ + p.name, + p.errors == null + ? { ..._void, idx: p.index } + : { + type: "lookupEntry" as const, + value: getLookupEntryDef(p.errors.type), + idx: p.index, + }, + ]), + ) as StringRecord< + ( + | VoidVar + | { + type: "lookupEntry" + value: LookupEntry + } + ) & { idx: number } + >, + byteLength: moduleErrorLength, + } + } + } + return null + }) + + function getModuleErrorLength(def: { + tag: "composite" + value: { + name: string | undefined + type: number + typeName: string | undefined + docs: string[] + }[] + }) { + const preChecks = + def.value.length === 2 && + def.value[0].name === "index" && + def.value[1].name === "error" + if (!preChecks) return null + + const index = getLookupEntryDef(def.value[0].type) + const error = getLookupEntryDef(def.value[1].type) + + return index.type === "primitive" && + index.value === "u8" && + error.type === "array" && + error.value.type === "primitive" && + error.value.value === "u8" + ? 1 + error.len + : null + } + + const getCall = () => { + if ("call" in metadata.extrinsic) { + return metadata.extrinsic.call + } + + const extrinsic = metadata.lookup[metadata.extrinsic.type] + const call = extrinsic?.params.find((p) => p.name === "Call") + + return call?.type ?? null + } + + return Object.assign(getLookupEntryDef, { metadata, call: getCall() }) +} diff --git a/packages/metadata-builders/src/with-cache.ts b/packages/metadata-builders/src/with-cache.ts new file mode 100644 index 0000000..16c115f --- /dev/null +++ b/packages/metadata-builders/src/with-cache.ts @@ -0,0 +1,44 @@ +import { LookupEntry } from "./lookups" + +type FnWithStack, T> = ( + input: LookupEntry, + cache: Map, + stack: Set, + ...rest: Other +) => T + +export const withCache = + , T>( + fn: FnWithStack, + onEnterCircular: ( + cacheGetter: () => T, + circular: LookupEntry, + ...rest: Other + ) => T, + onExitCircular: ( + outter: T, + inner: T, + circular: LookupEntry, + ...rest: Other + ) => T, + ): FnWithStack => + (input, cache, stack, ...rest) => { + const { id } = input + if (cache.has(id)) return cache.get(id)! + + if (stack.has(id)) { + const res = onEnterCircular(() => cache.get(id)!, input, ...rest) + cache.set(id, res) + return res + } + + stack.add(id) + let result = fn(input, cache, stack, ...rest) + stack.delete(id) + + if (cache.has(id)) + result = onExitCircular(result, cache.get(id)!, input, ...rest) + + cache.set(id, result) + return result + } diff --git a/packages/papi-utils/README.md b/packages/papi-utils/README.md index 6cbfc5e..9db8ef1 100644 --- a/packages/papi-utils/README.md +++ b/packages/papi-utils/README.md @@ -1 +1,12 @@ -# @polkadot-api/utils +# @pezkuwi/papi-utils + +**Source:** `src/` is [`@polkadot-api/utils`](https://www.npmjs.com/package/@polkadot-api/utils/v/0.2.0) 0.2.0 from +[polkadot-api](https://github.com/polkadot-api/polkadot-api) (MIT), with imports of +`@polkadot-api/*` renamed to `@pezkuwi/*`. `dist/` is its build; +`scripts/check-papi-sources.mjs` fails CI unless the source embedded in `dist/` equals `src/`. + +The upstream README follows. + +--- + +## @polkadot-api/utils diff --git a/packages/papi-utils/src/AbortError.ts b/packages/papi-utils/src/AbortError.ts new file mode 100644 index 0000000..236add9 --- /dev/null +++ b/packages/papi-utils/src/AbortError.ts @@ -0,0 +1,6 @@ +export class AbortError extends Error { + constructor() { + super("Abort Error") + this.name = "AbortError" + } +} diff --git a/packages/papi-utils/src/filterObject.ts b/packages/papi-utils/src/filterObject.ts new file mode 100644 index 0000000..e45b2ca --- /dev/null +++ b/packages/papi-utils/src/filterObject.ts @@ -0,0 +1,8 @@ +export function filterObject( + input: Record, + filterFn: (i: I, k: K) => boolean, +): Record { + return Object.fromEntries( + Object.entries(input).filter(([key, value]: any) => filterFn(value, key)), + ) as any +} diff --git a/packages/papi-utils/src/hex.ts b/packages/papi-utils/src/hex.ts new file mode 100644 index 0000000..047e0b0 --- /dev/null +++ b/packages/papi-utils/src/hex.ts @@ -0,0 +1,57 @@ +// https://jsben.ch/uWZw3 +const HEX_STR = "0123456789abcdef" +export function toHex(bytes: Uint8Array): string { + const result = new Array(bytes.length + 1) + + result[0] = "0x" + + for (let i = 0; i < bytes.length; ) { + const b = bytes[i++] + result[i] = HEX_STR[b >> 4] + HEX_STR[b & 15] + } + + return result.join("") +} + +// https://jsben.ch/URe1X +const HEX_MAP: Record = { + 0: 0, + 1: 1, + 2: 2, + 3: 3, + 4: 4, + 5: 5, + 6: 6, + 7: 7, + 8: 8, + 9: 9, + a: 10, + b: 11, + c: 12, + d: 13, + e: 14, + f: 15, + A: 10, + B: 11, + C: 12, + D: 13, + E: 14, + F: 15, +} +export function fromHex(hexString: string): Uint8Array { + const isOdd = hexString.length % 2 + const base = (hexString[1] === "x" ? 2 : 0) + isOdd + const nBytes = (hexString.length - base) / 2 + isOdd + const bytes = new Uint8Array(nBytes) + + if (isOdd) bytes[0] = 0 | HEX_MAP[hexString[2]] + + for (let i = 0; i < nBytes; ) { + const idx = base + i * 2 + const a = HEX_MAP[hexString[idx]] + const b = HEX_MAP[hexString[idx + 1]] + bytes[isOdd + i++] = (a << 4) | b + } + + return bytes +} diff --git a/packages/papi-utils/src/jsonPrint.ts b/packages/papi-utils/src/jsonPrint.ts new file mode 100644 index 0000000..5cecc9c --- /dev/null +++ b/packages/papi-utils/src/jsonPrint.ts @@ -0,0 +1,11 @@ +export const jsonPrint = (value: any, indent = 2) => + JSON.stringify( + value, + (_, v) => + typeof v === "bigint" + ? `${v}n` + : typeof v === "object" && typeof v?.asHex === "function" + ? v.asHex() + : v, + indent, + ) diff --git a/packages/papi-utils/src/mapObject.ts b/packages/papi-utils/src/mapObject.ts new file mode 100644 index 0000000..37bca21 --- /dev/null +++ b/packages/papi-utils/src/mapObject.ts @@ -0,0 +1,29 @@ +export function mapObject( + input: Record, + mapper: (i: I, k: K) => O, +): Record + +export function mapObject( + input: Record, + mapper: (i: I, k?: K) => O, +): Record { + return Object.fromEntries( + Object.entries(input).map( + ([key, value]: any) => [key, mapper(value, key)] as const, + ), + ) as any +} + +export type StringRecord = { + [Sym: symbol]: never + [Num: number]: never + [Str: string]: T +} + +export const mapStringRecord = ( + input: StringRecord, + mapper: (value: I, key: string) => O, +): StringRecord => + Object.fromEntries( + Object.entries(input).map(([key, value]) => [key, mapper(value, key)]), + ) as StringRecord diff --git a/packages/papi-utils/src/mergeUint8.ts b/packages/papi-utils/src/mergeUint8.ts new file mode 100644 index 0000000..5955efc --- /dev/null +++ b/packages/papi-utils/src/mergeUint8.ts @@ -0,0 +1,25 @@ +// TODO: deprecate spread overload in papi v2 + +interface MergeUint8 { + /** + * @deprecated This overload will be removed in PAPI v2. Migrate as + * follows: + * mergeUint8(arr1, arr2) => mergeUint8([arr1, arr2]) + */ + (...inputs: Array): Uint8Array + (inputs: Array): Uint8Array +} + +export const mergeUint8: MergeUint8 = (...i) => { + const inputs = (Array.isArray(i[0]) ? i[0] : i) as Uint8Array[] + const totalLen = inputs.reduce((acc, a) => acc + a.byteLength, 0) + const result = new Uint8Array(totalLen) + + for (let idx = 0, at = 0; idx < inputs.length; idx++) { + const current = inputs[idx] + result.set(current, at) + at += current.byteLength + } + + return result +} diff --git a/packages/papi-utils/src/noop.ts b/packages/papi-utils/src/noop.ts new file mode 100644 index 0000000..d1e8c7e --- /dev/null +++ b/packages/papi-utils/src/noop.ts @@ -0,0 +1 @@ +export const noop: () => void = Function.prototype as any diff --git a/scripts/check-papi-sources.mjs b/scripts/check-papi-sources.mjs new file mode 100644 index 0000000..c0502b9 --- /dev/null +++ b/scripts/check-papi-sources.mjs @@ -0,0 +1,71 @@ +// Copyright 2017-2026 @pezkuwi/api authors & contributors +// SPDX-License-Identifier: Apache-2.0 + +// papi-utils, bizinikiwi-bindings, metadata-builders and merkleize-metadata +// ship committed dist/ builds of upstream polkadot-api packages. Every source +// map in dist/ embeds the TypeScript it was compiled from; this check fails +// unless that embedded source is exactly the src/ committed beside it, so a +// published dist/ can only come from source held in this repository. + +import fs from 'node:fs'; +import path from 'node:path'; + +const PACKAGES = ['papi-utils', 'bizinikiwi-bindings', 'metadata-builders', 'merkleize-metadata']; +const failures = []; + +function walk (dir) { + return fs.readdirSync(dir, { withFileTypes: true }).flatMap((e) => { + const p = path.join(dir, e.name); + + return e.isDirectory() ? walk(p) : [p]; + }); +} + +for (const name of PACKAGES) { + const pkg = path.join('packages', name); + const srcDir = path.join(pkg, 'src'); + const embedded = new Map(); + + for (const map of walk(path.join(pkg, 'dist')).filter((f) => f.endsWith('.map'))) { + const { sources = [], sourcesContent = [] } = JSON.parse(fs.readFileSync(map, 'utf8')); + + sources.forEach((source, i) => { + const resolved = path.normalize(path.join(path.dirname(map), source)); + const rel = resolved.slice(resolved.indexOf(`${path.sep}src${path.sep}`) + 1); + const content = sourcesContent[i]; + + if (content == null) { + failures.push(`${map}: no embedded source for ${source}`); + } else if (embedded.has(rel) && embedded.get(rel) !== content) { + failures.push(`${map}: ${rel} differs between source maps`); + } else { + embedded.set(rel, content); + } + }); + } + + const committed = new Set(walk(srcDir).map((f) => path.relative(pkg, f))); + + for (const [rel, content] of embedded) { + const file = path.join(pkg, rel); + + if (!committed.has(rel)) { + failures.push(`${file}: in dist/ source maps but not committed`); + } else if (fs.readFileSync(file, 'utf8') !== content) { + failures.push(`${file}: differs from the source embedded in dist/`); + } + + committed.delete(rel); + } + + for (const rel of committed) { + failures.push(`${path.join(pkg, rel)}: committed but not part of the dist/ build`); + } + + console.log(`${name}: ${embedded.size} source files match dist/`); +} + +if (failures.length) { + console.error(failures.join('\n')); + process.exit(1); +} diff --git a/tsconfig.json b/tsconfig.json index 8dc318d..d1a7615 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,6 +16,11 @@ // these include the different-chain augmentations, which then overrides // the bizinikiwi base. TS only allow single augmentations, so exclude here "packages/api-augment/src/zagros/*.ts", - "packages/api-augment/src/pezkuwi/*.ts" + "packages/api-augment/src/pezkuwi/*.ts", + // upstream polkadot-api sources behind committed dist/ builds; checked by scripts/check-papi-sources.mjs + "packages/bizinikiwi-bindings/src/**/*", + "packages/merkleize-metadata/src/**/*", + "packages/metadata-builders/src/**/*", + "packages/papi-utils/src/**/*" ] }