chore: Integrate PeerDASKZG library

.github/workflows/ci.yml

@@ -71,6 +71,11 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+
+      - name: Set up Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: 1.78.0
 
       - name: Restore llvm-mingw (Windows) from cache
         if: runner.os == 'Windows'

Makefile

@@ -152,7 +152,7 @@ GIT_SUBMODULE_UPDATE := git $(GIT_SUBMODULE_CONFIG) submodule update --init --re
 else # "variables.mk" was included. Business as usual until the end of this file.
 
 # default target, because it's the first one that doesn't start with '.'
-all: | $(TOOLS) libnfuzz.so libnfuzz.a $(PLATFORM_SPECIFIC_TARGETS)
+all: | $(TOOLS) libnfuzz.so libnfuzz.a $(PLATFORM_SPECIFIC_TARGETS) build-rust-peerdas-kzg
 
 # must be included after the default target
 -include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
@@ -391,7 +391,7 @@ block_sim: | build deps
 
 DISABLE_TEST_FIXTURES_SCRIPT := 0
 # This parameter passing scheme is ugly, but short.
-test: | $(XML_TEST_BINARIES) $(TEST_BINARIES)
+test: | $(XML_TEST_BINARIES) $(TEST_BINARIES) test-rust-peerdas-kzg
 ifeq ($(DISABLE_TEST_FIXTURES_SCRIPT), 0)
 	V=$(V) scripts/setup_scenarios.sh
 endif
@@ -606,6 +606,13 @@ define CLEAN_NETWORK
 	rm -rf build/data/shared_$(1)*/*.log
 endef
 
+# Build the rust peerdas-kzg library
+build-rust-peerdas-kzg:
+	@./build_peerdas_lib.sh nimpeerdaskzg
+test-rust-peerdas-kzg: # | build deps
+	+ echo -e $(BUILD_MSG) "Running KZG fixture test" && \
+		$(ENV_SCRIPT) nim c -r -d:USE_NIMPEERDAS_KZG tests/consensus_spec/test_fixture_kzg.nim && \
+		echo -e $(BUILD_END_MSG) "KZG fixture test completed"
 ###
 ### Sepolia
 ###

beacon_chain/kzg.nim

@@ -0,0 +1,39 @@
+# beacon_chain
+# Copyright (c) 2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+
+import
+  std/os,
+  results,
+  kzg4844/kzg_ex,
+  ../../../nimpeerdaskzg/nim_peerdas_kzg/nim_peerdas_kzg
+
+
+from std/strutils import rsplit
+
+var ctx: nim_peerdas_kzg.KZGCtx
+
+proc initKZG*(): bool =
+  # TODO: no compilation flag here because c-kzg does more than peerdas functionality.   
+  ctx = newKZGCtx()
+  template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]
+  Kzg.loadTrustedSetup(
+    sourceDir &
+      "/../vendor/nim-kzg4844/kzg4844/csources/src/trusted_setup.txt").isOk
+
+proc freeKZG*(): Result[void, string] =
+  # TODO: add code to free nim_peerdas_kzg. Removed due to nim not allowing
+  # TODO; the particular destory function being called.
+  Kzg.freeTrustedSetup()
+
+proc blobToKZGCommitment*(blob : array[131_072, byte]): Result[array[48, byte], string] =
+  when defined(USE_NIMPEERDAS_KZG):
+    let commitment = ?ctx.blobToKzgCommitment(Blob(bytes: blob))
+    ok(commitment.bytes)
+  else:
+    kzg_ex.blobToKZGCommitment(blob)

build_peerdas_lib.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2024 Status Research & Development GmbH. Licensed under
+# either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+# Current usage: ./build_peerdas_lib.sh nimpeerdaskzg
+
+set -eu  # Exit immediately if a command exits with a non-zero status.
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <destination_folder>"
+    exit 1
+fi
+
+
+DESTINATION_FOLDER=$1 # Destination folder to copy the built nim library with the static lib include to
+
+COMMIT_HASH="4205937b69945f23731d90ba2970f19fa4a5e06b" # commit to checkout rust lib at
+REPO_URL="https://github.com/crate-crypto/peerdas-kzg"
+
+echo "Building peerdas-kzg with commit hash: $COMMIT_HASH and destination: $DESTINATION_FOLDER"
+
+TEMP_DIR=$(mktemp -d)
+echo "Created temporary directory: $TEMP_DIR"
+
+echo "Cloning repository..."
+git clone "$REPO_URL" "$TEMP_DIR"
+cd "$TEMP_DIR"
+
+echo "Checking out commit: $COMMIT_HASH"
+git checkout "$COMMIT_HASH"
+
+# Check if the user is on Windows and install gnu target instead of msvc
+if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then
+    echo "Detected Windows environment. Adding x86_64-pc-windows-gnu target..."
+    rustup target add x86_64-pc-windows-gnu
+fi
+
+echo "Building Rust Library: Running ./scripts/compile.sh nim"
+if [ -f "./scripts/compile.sh" ]; then
+    ./scripts/compile.sh nim
+else
+    echo "Error: ./scripts/compile.sh not found"
+    exit 1
+fi
+
+if [ ! -d "bindings/nim" ]; then
+    echo "Error: bindings/nim directory not found"
+    exit 1
+fi
+
+# Move back to the original directory that the script was called from
+cd -
+
+echo "Creating destination folder: $DESTINATION_FOLDER"
+mkdir -p "$DESTINATION_FOLDER"
+
+# Copy the nim code to the destination folder (includes the built static lib) 
+echo "Copying contents of bindings/nim/nim_code to $DESTINATION_FOLDER"
+cp -a "$TEMP_DIR/bindings/nim/nim_code/." "$DESTINATION_FOLDER"
+
+# Clean up the temporary directory
+rm -rf "$TEMP_DIR"
+
+echo "Successfully built peerdas-kzg library and copied the nimble package to $DESTINATION_FOLDER"

tests/consensus_spec/test_fixture_kzg.nim

@@ -11,13 +11,17 @@
 import
   std/json,
   yaml,
-  kzg4844/kzg_ex,
+  ../../../beacon_chain/kzg,
   stint,
   chronicles,
   stew/[byteutils, results],
   ../testutil,
   ./fixtures_utils, ./os_ops
 
+# TODO: don't import blobToKzgCommitment since we 
+# TODO: are using the beacon_chain/kzg module for it
+import kzg4844/kzg_ex except blobToKzgCommitment
+
 from std/sequtils import anyIt, mapIt, toSeq
 from std/strutils import rsplit
 
@@ -40,10 +44,7 @@ func fromHex[N: static int](s: string): Opt[array[N, byte]] =
     Opt.none array[N, byte]
 
 block:
-  template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]
-  doAssert Kzg.loadTrustedSetup(
-    sourceDir &
-      "/../../vendor/nim-kzg4844/kzg4844/csources/src/trusted_setup.txt").isOk
+  doAssert initKZG()
 
 proc runBlobToKzgCommitmentTest(suiteName, suitePath, path: string) =
   let relativePathComponent = path.relativeTestPathComponent(suitePath)
@@ -399,4 +400,4 @@ suite suiteName:
     for kind, path in walkDir(testsDir, relative = true, checkDir = true):
       runVerifyCellKzgProofBatchTest(suiteName, testsDir, testsDir/path)
 
-doAssert Kzg.freeTrustedSetup().isOk
+doAssert freeKZG().isOk