From 5e70ebe267e14d55980781c92a850d673bfc030c Mon Sep 17 00:00:00 2001 From: zz-sol Date: Wed, 18 Mar 2026 15:33:47 -0400 Subject: [PATCH] integrate cuda accelerator for curve25519 (#3) * integrate cuda * CI --- Cargo.lock | 218 +++++++++++++++++++++-- curve25519-cuda/Cargo.toml | 21 ++- curve25519-cuda/benches/msm.rs | 114 ++++++++++++ curve25519-cuda/build.rs | 72 ++++++++ curve25519-cuda/config_cuda.sh | 47 +++++ curve25519-cuda/src/lib.rs | 220 +++++++++++++++++++++-- curve25519-cuda/src/tests.rs | 311 +++++++++++++++++++++++++++++++++ 7 files changed, 978 insertions(+), 25 deletions(-) create mode 100644 curve25519-cuda/benches/msm.rs create mode 100644 curve25519-cuda/build.rs create mode 100755 curve25519-cuda/config_cuda.sh create mode 100644 curve25519-cuda/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 065aa2f..0ee5fae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,6 +109,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cc" +version = "1.2.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -272,10 +282,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "curve25519-cuda" -version = "0.1.0" - [[package]] name = "digest" version = "0.11.1" @@ -309,7 +315,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -318,6 +324,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + [[package]] name = "fnv" version = "1.0.7" @@ -336,6 +348,17 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -406,6 +429,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "hybrid-array" version = "0.4.8" @@ -441,7 +473,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -481,6 +513,12 @@ version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -607,7 +645,7 @@ dependencies = [ "bitflags", "num-traits", "rand 0.9.2", - "rand_chacha", + "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", "rusty-fork", @@ -648,13 +686,24 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -669,6 +718,16 @@ dependencies = [ "rand_core 0.10.0", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -679,6 +738,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rand_core" version = "0.9.5" @@ -799,6 +867,19 @@ dependencies = [ "subtle", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -808,8 +889,8 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", - "windows-sys", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", ] [[package]] @@ -899,6 +980,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "solana-curve25519" version = "0.1.0" @@ -922,6 +1009,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "solana-curve25519-cuda" +version = "0.1.0" +dependencies = [ + "cc", + "criterion", + "rand 0.8.5", + "rand_chacha 0.3.1", + "rand_core 0.6.4", + "rayon", + "solana-curve25519", + "which", +] + [[package]] name = "solana-curve25519-derive" version = "0.1.0" @@ -974,8 +1075,8 @@ dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", - "rustix", - "windows-sys", + "rustix 1.1.4", + "windows-sys 0.61.2", ] [[package]] @@ -1031,6 +1132,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasip2" version = "1.0.2+wasi-0.2.9" @@ -1138,13 +1245,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "winapi-util" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1153,6 +1272,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1162,6 +1290,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "wit-bindgen" version = "0.51.0" diff --git a/curve25519-cuda/Cargo.toml b/curve25519-cuda/Cargo.toml index 86b41b2..69ab68e 100644 --- a/curve25519-cuda/Cargo.toml +++ b/curve25519-cuda/Cargo.toml @@ -1,6 +1,25 @@ [package] -name = "curve25519-cuda" +name = "solana-curve25519-cuda" version = "0.1.0" edition = "2024" +authors = ["Anza Cryptography Team"] [dependencies] +# Keep the dependency key as "curve25519" so that all existing `use curve25519::…` +# references in source files remain valid; the actual package is curve25519-sol. +curve25519 = { path = "../curve25519", package = "solana-curve25519" } +rand = "0.8" +rand_chacha = "0.3" +rand_core = "0.6" +rayon = "1.10" + +[build-dependencies] +cc = "1.0" +which = "4.4" + +[dev-dependencies] +criterion = "0.5" + +[[bench]] +name = "msm" +harness = false diff --git a/curve25519-cuda/benches/msm.rs b/curve25519-cuda/benches/msm.rs new file mode 100644 index 0000000..22475d9 --- /dev/null +++ b/curve25519-cuda/benches/msm.rs @@ -0,0 +1,114 @@ +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use curve25519::edwards::EdwardsPoint; +use curve25519::scalar::Scalar; +use curve25519::short_weierstrass::SwPoint; +use rayon::prelude::*; + +fn cpu_msm(points: &[SwPoint], scalars: &[Scalar]) -> SwPoint { + use curve25519::traits::{Identity, VartimeMultiscalarMul}; + assert_eq!(points.len(), scalars.len()); + + let ed_points: Vec = points + .par_iter() + .map(|point| point.to_edwards().expect("valid sw point")) + .collect(); + + let chunk_size = ed_points + .len() + .max(1) + .div_ceil(rayon::current_num_threads()); + let out = ed_points + .par_chunks(chunk_size) + .zip(scalars.par_chunks(chunk_size)) + .map(|(point_chunk, scalar_chunk)| { + EdwardsPoint::vartime_multiscalar_mul(scalar_chunk, point_chunk) + }) + .reduce(EdwardsPoint::identity, |acc, partial| acc + partial); + + SwPoint::from_edwards(&out) +} + +fn splitmix64(state: &mut u64) -> u64 { + let mut z = { + *state = state.wrapping_add(0x9e3779b97f4a7c15); + *state + }; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58476d1ce4e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d049bb133111eb); + z ^ (z >> 31) +} + +fn bytes_from_seed(seed: u64) -> [u8; 32] { + let mut state = seed; + let mut out = [0u8; 32]; + for chunk in out.chunks_exact_mut(8) { + let v = splitmix64(&mut state).to_le_bytes(); + chunk.copy_from_slice(&v); + } + out +} + +fn build_inputs(log_n: usize) -> (Vec, Vec) { + let npoints = 1usize << log_n; + let base = curve25519::constants::ED25519_BASEPOINT_POINT; + let pairs: Vec<(SwPoint, Scalar)> = (0..npoints) + .into_par_iter() + .map(|i| { + let seed = ((log_n as u64) << 32) ^ (i as u64); + let s_bytes = bytes_from_seed(seed ^ 0x6a09e667f3bcc909); + let p_scalar = Scalar::from_bytes_mod_order(s_bytes); + let p_ed = base * p_scalar; + let point = SwPoint::from_edwards(&p_ed); + + let sc_bytes = bytes_from_seed(seed ^ 0xbb67ae8584caa73b); + let scalar = Scalar::from_bytes_mod_order(sc_bytes); + (point, scalar) + }) + .collect(); + + let mut points = Vec::with_capacity(npoints); + let mut scalars = Vec::with_capacity(npoints); + for (point, scalar) in pairs { + points.push(point); + scalars.push(scalar); + } + + (points, scalars) +} + +fn bench_cpu_vs_gpu(c: &mut Criterion) { + let logs = [1usize, 2, 4, 8, 12, 16, 20]; + + for &log_n in &logs { + let (points, scalars) = build_inputs(log_n); + + let mut cpu_group = c.benchmark_group("msm_cpu"); + cpu_group.sample_size(10); + cpu_group.bench_with_input( + BenchmarkId::from_parameter(format!("log2={}", log_n)), + &log_n, + |b, _| { + b.iter(|| { + let _ = cpu_msm(&points, &scalars); + }); + }, + ); + cpu_group.finish(); + + let mut gpu_group = c.benchmark_group("msm_gpu"); + gpu_group.sample_size(10); + gpu_group.bench_with_input( + BenchmarkId::from_parameter(format!("log2={}", log_n)), + &log_n, + |b, _| { + b.iter(|| { + let _ = solana_curve25519_cuda::msm_curve25519_gpu(&points, &scalars); + }); + }, + ); + gpu_group.finish(); + } +} + +criterion_group!(benches, bench_cpu_vs_gpu); +criterion_main!(benches); diff --git a/curve25519-cuda/build.rs b/curve25519-cuda/build.rs new file mode 100644 index 0000000..ff4a5ab --- /dev/null +++ b/curve25519-cuda/build.rs @@ -0,0 +1,72 @@ +use std::env; +use std::path::PathBuf; +use std::process::Command; + +fn main() { + println!("cargo:rustc-check-cfg=cfg(curve25519_cuda)"); + let target = env::var("TARGET").unwrap_or_default(); + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let sppark_dir = manifest_dir.join("..").join("sppark"); + let blst_dir = sppark_dir.join("blst"); + let blst_lib = blst_dir.join("libblst.a"); + let cuda_file = sppark_dir + .join("poc") + .join("msm-cuda") + .join("cuda") + .join("pippenger_curve25519.cu"); + let all_gpus = sppark_dir.join("util").join("all_gpus.cpp"); + let blst_include = blst_dir.join("src"); + + let nvcc = env::var("NVCC") + .ok() + .or_else(|| which::which("nvcc").ok().map(|p| p.display().to_string())); + if nvcc.is_none() { + return; + } + + if !blst_lib.exists() { + let status = Command::new("./build.sh") + .current_dir(&blst_dir) + .status() + .expect("failed to run blst build.sh"); + if !status.success() { + panic!("blst build failed"); + } + } + + if env::var("CXX").is_err() { + let cxx = which::which("g++-12") + .or_else(|_| which::which("g++")) + .ok() + .map(|p| p.display().to_string()); + if let Some(cxx) = cxx { + unsafe { + env::set_var("CXX", &cxx); + if !target.is_empty() { + env::set_var(format!("CXX_{}", target), &cxx); + } + } + } + } + + let mut nvcc_build = cc::Build::new(); + nvcc_build + .cuda(true) + .flag("-std=c++17") + .flag("-allow-unsupported-compiler") + .flag("-Xcompiler") + .flag("-std=gnu++17") + .define("__ADX__", None) + .include(&sppark_dir) + .include(&blst_include) + .file(&cuda_file) + .file(&all_gpus); + nvcc_build.compile("curve25519_cuda_msm"); + + println!("cargo:rustc-link-search=native={}", blst_dir.display()); + println!("cargo:rustc-link-lib=static=blst"); + println!("cargo:rustc-link-lib=dylib=cudart"); + println!("cargo:rustc-cfg=curve25519_cuda"); + println!("cargo:rerun-if-changed={}", cuda_file.display()); + println!("cargo:rerun-if-changed={}", all_gpus.display()); +} diff --git a/curve25519-cuda/config_cuda.sh b/curve25519-cuda/config_cuda.sh new file mode 100755 index 0000000..0e25449 --- /dev/null +++ b/curve25519-cuda/config_cuda.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SPPARK_DIR="${ROOT_DIR}/sppark" +REMOTE_URL="https://github.com/zz-sol/sppark.git" +REMOTE_BRANCH="ed25519" +BLST_DIR="${SPPARK_DIR}/blst" +BLST_REMOTE_URL="https://github.com/supranational/blst.git" + +if [[ -d "${SPPARK_DIR}/.git" ]]; then + echo "Updating sppark in ${SPPARK_DIR}" + git -C "${SPPARK_DIR}" remote set-url origin "${REMOTE_URL}" >/dev/null 2>&1 || true + git -C "${SPPARK_DIR}" fetch origin + git -C "${SPPARK_DIR}" checkout "${REMOTE_BRANCH}" + git -C "${SPPARK_DIR}" pull --ff-only origin "${REMOTE_BRANCH}" +elif [[ -d "${SPPARK_DIR}" ]]; then + ts="$(date +%Y%m%d_%H%M%S)" + backup="${SPPARK_DIR}.backup.${ts}" + echo "Existing sppark directory found. Moving to ${backup}" + mv "${SPPARK_DIR}" "${backup}" + git clone --branch "${REMOTE_BRANCH}" --depth 1 "${REMOTE_URL}" "${SPPARK_DIR}" +else + git clone --branch "${REMOTE_BRANCH}" --depth 1 "${REMOTE_URL}" "${SPPARK_DIR}" +fi + +if [[ -d "${BLST_DIR}/.git" ]]; then + echo "Updating blst in ${BLST_DIR}" + git -C "${BLST_DIR}" remote set-url origin "${BLST_REMOTE_URL}" >/dev/null 2>&1 || true + git -C "${BLST_DIR}" fetch origin + git -C "${BLST_DIR}" pull --ff-only +elif [[ -d "${BLST_DIR}" ]]; then + ts="$(date +%Y%m%d_%H%M%S)" + backup="${BLST_DIR}.backup.${ts}" + echo "Existing blst directory found. Moving to ${backup}" + mv "${BLST_DIR}" "${backup}" + git clone --depth 1 "${BLST_REMOTE_URL}" "${BLST_DIR}" +else + git clone --depth 1 "${BLST_REMOTE_URL}" "${BLST_DIR}" +fi + +if [[ ! -f "${BLST_DIR}/build.sh" ]]; then + echo "Expected blst build.sh at ${BLST_DIR}/build.sh but it is missing." + exit 1 +fi + +echo "sppark ready at ${SPPARK_DIR}" diff --git a/curve25519-cuda/src/lib.rs b/curve25519-cuda/src/lib.rs index b93cf3f..5f3e222 100644 --- a/curve25519-cuda/src/lib.rs +++ b/curve25519-cuda/src/lib.rs @@ -1,14 +1,212 @@ -pub fn add(left: u64, right: u64) -> u64 { - left + right +use core::ffi::c_char; +use curve25519::scalar::Scalar; +use curve25519::short_weierstrass::SwPoint; +#[cfg(curve25519_cuda)] +use std::ffi::CStr; + +/// Affine point layout compatible with SPPARK `affine_t` (host-side). +/// +/// Coordinates are canonical little-endian limb arrays; SPPARK expects +/// Montgomery form, so call-side conversion is still required. +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SpparkAffine { + pub x: [u64; 4], + pub y: [u64; 4], +} + +/// Scalar layout compatible with SPPARK `fr_t` (host-side). +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SpparkScalar { + pub limbs: [u64; 4], +} + +/// Byte-serialized affine layout for SPPARK conversion helpers. +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SpparkAffineBytes { + pub x: [u8; 32], + pub y: [u8; 32], +} + +/// Byte-serialized scalar layout for SPPARK conversion helpers. +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SpparkScalarBytes { + pub s: [u8; 32], +} + +/// Error type matching SPPARK's FFI error layout. +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SpparkError { + pub code: i32, + pub str: *mut c_char, +} + +fn le_bytes_to_u64x4(bytes: [u8; 32]) -> [u64; 4] { + let mut limbs = [0u64; 4]; + for (i, limb) in limbs.iter_mut().enumerate() { + let mut buf = [0u8; 8]; + buf.copy_from_slice(&bytes[i * 8..(i + 1) * 8]); + *limb = u64::from_le_bytes(buf); + } + limbs +} + +/// Convert a short Weierstrass point into SPPARK affine layout. +/// +/// The identity is encoded as (0, 0) to match SPPARK `affine_t` rules. +pub fn sw_point_to_sppark_affine(point: &SwPoint) -> SpparkAffine { + match point.to_affine_le_bytes() { + Some((x, y)) => SpparkAffine { + x: le_bytes_to_u64x4(x), + y: le_bytes_to_u64x4(y), + }, + None => SpparkAffine { + x: [0u64; 4], + y: [0u64; 4], + }, + } +} + +/// Convert a scalar into SPPARK scalar limb layout. +pub fn scalar_to_sppark_scalar(scalar: &Scalar) -> SpparkScalar { + let bytes = scalar.to_bytes(); + SpparkScalar { + limbs: le_bytes_to_u64x4(bytes), + } +} + +/// Convert a short Weierstrass point into byte-serialized affine layout. +pub fn sw_point_to_sppark_affine_bytes(point: &SwPoint) -> SpparkAffineBytes { + match point.to_affine_le_bytes() { + Some((x, y)) => SpparkAffineBytes { x, y }, + None => SpparkAffineBytes { + x: [0u8; 32], + y: [0u8; 32], + }, + } +} + +/// Convert a scalar into byte-serialized layout. +pub fn scalar_to_sppark_scalar_bytes(scalar: &Scalar) -> SpparkScalarBytes { + SpparkScalarBytes { + s: scalar.to_bytes(), + } +} + +#[cfg(curve25519_cuda)] +unsafe extern "C" { + pub fn mult_pippenger_curve25519_bytes_affine( + out: *mut SpparkAffineBytes, + points: *const SpparkAffineBytes, + npoints: usize, + scalars: *const SpparkScalarBytes, + ) -> SpparkError; +} + +/// Execute MSM on the CUDA backend using byte-serialized affine inputs. +pub fn msm_curve25519_gpu(points: &[SwPoint], scalars: &[Scalar]) -> Result { + let out = msm_curve25519_gpu_bytes(points, scalars)?; + if out.x == [0u8; 32] && out.y == [0u8; 32] { + return Ok(SwPoint::Identity); + } + SwPoint::from_affine_le_bytes(out.x, out.y) + .ok_or_else(|| "invalid point returned from CUDA".to_string()) +} + +fn msm_curve25519_cpu(points: &[SwPoint], scalars: &[Scalar]) -> Result { + use curve25519::edwards::EdwardsPoint; + use curve25519::traits::Identity; + use curve25519::traits::VartimeMultiscalarMul; + use rayon::prelude::*; + + if points.len() != scalars.len() { + return Err("length mismatch".to_string()); + } + if points.is_empty() { + return Ok(SwPoint::Identity); + } + + let ed_points: Vec = points + .par_iter() + .map(|point| { + point + .to_edwards() + .ok_or_else(|| "invalid short-weierstrass point".to_string()) + }) + .collect::>()?; + + let chunk = 1024usize.max(points.len() / rayon::current_num_threads().max(1)); + let ed_out = ed_points + .par_chunks(chunk) + .zip(scalars.par_chunks(chunk)) + .map(|(pchunk, schunk)| EdwardsPoint::vartime_multiscalar_mul(schunk, pchunk)) + .reduce(EdwardsPoint::identity, |a, b| a + b); + Ok(SwPoint::from_edwards(&ed_out)) +} + +/// Execute MSM using CUDA when available, otherwise fall back to CPU in Rust. +pub fn msm_curve25519(points: &[SwPoint], scalars: &[Scalar]) -> Result { + if let Ok(out) = msm_curve25519_gpu(points, scalars) { + return Ok(out); + } + msm_curve25519_cpu(points, scalars) +} + +/// Execute MSM on the CUDA backend and return byte-serialized affine output. +pub fn msm_curve25519_gpu_bytes( + points: &[SwPoint], + scalars: &[Scalar], +) -> Result { + if points.len() != scalars.len() { + return Err("length mismatch".to_string()); + } + + #[cfg(not(curve25519_cuda))] + { + let _ = (points, scalars); + Err("CUDA backend not available".to_string()) + } + + #[cfg(curve25519_cuda)] + { + let point_bytes: Vec = + points.iter().map(sw_point_to_sppark_affine_bytes).collect(); + let scalar_bytes: Vec = + scalars.iter().map(scalar_to_sppark_scalar_bytes).collect(); + + let mut out = SpparkAffineBytes { + x: [0u8; 32], + y: [0u8; 32], + }; + + let err = unsafe { + mult_pippenger_curve25519_bytes_affine( + &mut out as *mut _, + point_bytes.as_ptr(), + point_bytes.len(), + scalar_bytes.as_ptr(), + ) + }; + + if err.code != 0 { + let message = if !err.str.is_null() { + unsafe { CStr::from_ptr(err.str) } + .to_str() + .unwrap_or("sppark error") + .to_string() + } else { + format!("sppark error code {}", err.code) + }; + return Err(message); + } + + Ok(out) + } } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } -} +mod tests; diff --git a/curve25519-cuda/src/tests.rs b/curve25519-cuda/src/tests.rs new file mode 100644 index 0000000..2881f51 --- /dev/null +++ b/curve25519-cuda/src/tests.rs @@ -0,0 +1,311 @@ +use super::{ + msm_curve25519, scalar_to_sppark_scalar, scalar_to_sppark_scalar_bytes, + sw_point_to_sppark_affine, sw_point_to_sppark_affine_bytes, +}; +use curve25519::edwards::EdwardsPoint; +use curve25519::scalar::Scalar; +use curve25519::short_weierstrass::SwPoint; +#[cfg(curve25519_cuda)] +use rayon::prelude::*; + +fn cpu_expected(points: &[SwPoint], scalars: &[Scalar]) -> SwPoint { + use curve25519::traits::VartimeMultiscalarMul; + let mut ed_points = Vec::with_capacity(points.len()); + for point in points { + ed_points.push(point.to_edwards().expect("valid sw point")); + } + let out = EdwardsPoint::vartime_multiscalar_mul(scalars, &ed_points); + SwPoint::from_edwards(&out) +} + +#[test] +fn sppark_affine_identity_is_zero() { + let affine = sw_point_to_sppark_affine(&SwPoint::Identity); + assert_eq!(affine.x, [0u64; 4]); + assert_eq!(affine.y, [0u64; 4]); +} + +#[test] +fn sppark_scalar_one_is_one() { + let scalar = scalar_to_sppark_scalar(&Scalar::ONE); + assert_eq!(scalar.limbs, [1u64, 0, 0, 0]); +} + +#[test] +fn sppark_affine_bytes_identity_is_zero() { + let affine = sw_point_to_sppark_affine_bytes(&SwPoint::Identity); + assert_eq!(affine.x, [0u8; 32]); + assert_eq!(affine.y, [0u8; 32]); +} + +#[test] +fn sppark_scalar_bytes_one_is_one() { + let scalar = scalar_to_sppark_scalar_bytes(&Scalar::ONE); + let mut expected = [0u8; 32]; + expected[0] = 1; + assert_eq!(scalar.s, expected); +} + +#[test] +fn msm_curve25519_rejects_length_mismatch() { + let base = SwPoint::from_edwards(&curve25519::constants::ED25519_BASEPOINT_POINT); + let points = vec![base, base]; + let scalars = vec![Scalar::ONE]; + let err = msm_curve25519(&points, &scalars).unwrap_err(); + assert!(err.contains("length mismatch")); +} + +#[test] +fn msm_curve25519_empty_returns_identity() { + let out = msm_curve25519(&[], &[]).expect("empty msm"); + assert_eq!(out, SwPoint::Identity); +} + +#[test] +fn msm_curve25519_matches_cpu_small() { + let base = SwPoint::from_edwards(&curve25519::constants::ED25519_BASEPOINT_POINT); + let points = vec![base, base, base.add(&base)]; + let scalars = vec![Scalar::ONE, Scalar::from(2u64), Scalar::from(3u64)]; + let out = msm_curve25519(&points, &scalars).expect("msm"); + let expected = cpu_expected(&points, &scalars); + assert_eq!(out, expected); +} + +#[test] +fn msm_curve25519_high_bit_scalars() { + let base = SwPoint::from_edwards(&curve25519::constants::ED25519_BASEPOINT_POINT); + let points = vec![base, base.add(&base)]; + let mut high = [0u8; 32]; + high[31] = 0x80; + let scalars = vec![Scalar::from_bytes_mod_order(high), Scalar::from(5u64)]; + let out = msm_curve25519(&points, &scalars).expect("msm"); + let expected = cpu_expected(&points, &scalars); + assert_eq!(out, expected); +} + +#[test] +fn msm_curve25519_infinity_points() { + let base = SwPoint::from_edwards(&curve25519::constants::ED25519_BASEPOINT_POINT); + let points = vec![SwPoint::Identity, base, SwPoint::Identity]; + let scalars = vec![Scalar::from(7u64), Scalar::from(9u64), Scalar::from(11u64)]; + let out = msm_curve25519(&points, &scalars).expect("msm"); + let expected = cpu_expected(&points, &scalars); + assert_eq!(out, expected); +} + +#[cfg(curve25519_cuda)] +fn edwards_msm( + points: &[curve25519::edwards::EdwardsPoint], + scalars: &[Scalar], +) -> curve25519::edwards::EdwardsPoint { + use curve25519::traits::Identity; + + let mut acc = curve25519::edwards::EdwardsPoint::identity(); + for (p, s) in points.iter().zip(scalars.iter()) { + acc = acc + (p * s); + } + acc +} + +#[cfg(curve25519_cuda)] +fn splitmix64(state: &mut u64) -> u64 { + let mut z = { + *state = state.wrapping_add(0x9e3779b97f4a7c15); + *state + }; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58476d1ce4e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d049bb133111eb); + z ^ (z >> 31) +} + +#[cfg(curve25519_cuda)] +fn bytes_from_seed(seed: u64) -> [u8; 32] { + let mut state = seed; + let mut out = [0u8; 32]; + for chunk in out.chunks_exact_mut(8) { + let v = splitmix64(&mut state).to_le_bytes(); + chunk.copy_from_slice(&v); + } + out +} + +#[cfg(curve25519_cuda)] +fn negate_field_bytes(y: [u8; 32]) -> [u8; 32] { + if y == [0u8; 32] { + return y; + } + let p = [ + 0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x7f, + ]; + let mut out = [0u8; 32]; + let mut borrow = 0u16; + for i in 0..32 { + let pi = p[i] as i16; + let yi = y[i] as i16; + let mut val = pi - yi - borrow as i16; + if val < 0 { + val += 256; + borrow = 1; + } else { + borrow = 0; + } + out[i] = val as u8; + } + out +} + +#[cfg(curve25519_cuda)] +fn affine_bytes_match(out: &crate::SpparkAffineBytes, expected: &crate::SpparkAffineBytes) -> bool { + if out.x != expected.x { + return false; + } + if out.y == expected.y { + return true; + } + out.y == negate_field_bytes(expected.y) +} + +#[test] +#[cfg(curve25519_cuda)] +fn cuda_msm_matches_cpu() { + use super::msm_curve25519_gpu_bytes; + use curve25519::constants; + + let p_ed = constants::ED25519_BASEPOINT_POINT; + let p = SwPoint::from_edwards(&p_ed); + let points = vec![p, p]; + let scalars = vec![Scalar::ONE, Scalar::from(2u64)]; + + let expected_ed = edwards_msm(&[p_ed, p_ed], &scalars); + let expected = SwPoint::from_edwards(&expected_ed); + let expected_bytes = sw_point_to_sppark_affine_bytes(&expected); + let out = msm_curve25519_gpu_bytes(&points, &scalars).expect("cuda msm"); + + assert!( + affine_bytes_match(&out, &expected_bytes), + "unexpected MSM output" + ); +} + +#[test] +#[cfg(curve25519_cuda)] +fn cuda_single_scalar_mul() { + use super::msm_curve25519_gpu_bytes; + use curve25519::constants; + + let base_ed = constants::ED25519_BASEPOINT_POINT; + let base = SwPoint::from_edwards(&base_ed); + + // Test basepoint with small scalars + let scalar3 = Scalar::from(3u64); + + let expected_ed_3 = base_ed * scalar3; + let expected_3 = SwPoint::from_edwards(&expected_ed_3); + let expected_bytes_3 = sw_point_to_sppark_affine_bytes(&expected_3); + let out_3 = msm_curve25519_gpu_bytes(&[base], &[scalar3]).expect("cuda msm"); + + assert!( + affine_bytes_match(&out_3, &expected_bytes_3), + "BASEPOINT * 3 should match" + ); + + // Test basepoint with random scalar + let scalar_random = Scalar::from_bytes_mod_order(bytes_from_seed(0xabcdefu64)); + + let expected_ed_base = base_ed * scalar_random; + let expected_base = SwPoint::from_edwards(&expected_ed_base); + let expected_bytes_base = sw_point_to_sppark_affine_bytes(&expected_base); + let out_base = msm_curve25519_gpu_bytes(&[base], &[scalar_random]).expect("cuda msm"); + + assert!( + affine_bytes_match(&out_base, &expected_bytes_base), + "BASEPOINT * scalar should match" + ); + + // Generate a random point + let s = Scalar::from_bytes_mod_order(bytes_from_seed(0x1337u64)); + let p_ed = base_ed * s; + let p = SwPoint::from_edwards(&p_ed); + assert_eq!(p.to_edwards(), Some(p_ed), "round-trip failed"); + + // Test random point with scalar = 1 + let scalar = Scalar::ONE; + let expected_ed = p_ed * scalar; + let expected = SwPoint::from_edwards(&expected_ed); + let expected_bytes = sw_point_to_sppark_affine_bytes(&expected); + let out = msm_curve25519_gpu_bytes(&[p], &[scalar]).expect("cuda msm"); + assert!( + affine_bytes_match(&out, &expected_bytes), + "P * 1 should equal P" + ); + + // Test random point with scalar = 2 + let scalar2 = Scalar::from(2u64); + let expected_ed2 = p_ed * scalar2; + let expected2 = SwPoint::from_edwards(&expected_ed2); + let expected_bytes2 = sw_point_to_sppark_affine_bytes(&expected2); + let out2 = msm_curve25519_gpu_bytes(&[p], &[scalar2]).expect("cuda msm"); + assert!( + affine_bytes_match(&out2, &expected_bytes2), + "P * 2 should match" + ); + + // Test random point with random scalar (the failing case) + let scalar3 = Scalar::from_bytes_mod_order(bytes_from_seed(0xdeadbeefu64)); + let expected_ed3 = p_ed * scalar3; + let expected3 = SwPoint::from_edwards(&expected_ed3); + let expected_bytes3 = sw_point_to_sppark_affine_bytes(&expected3); + let out3 = msm_curve25519_gpu_bytes(&[p], &[scalar3]).expect("cuda msm"); + assert!( + affine_bytes_match(&out3, &expected_bytes3), + "P * scalar should match" + ); +} + +#[test] +#[cfg(curve25519_cuda)] +fn cuda_msm_randomized_batches() { + use super::msm_curve25519_gpu_bytes; + use curve25519::constants; + + let base = constants::ED25519_BASEPOINT_POINT; + let sizes = [1usize, 2, 4, 8, 16]; + + for &n in &sizes { + let pairs: Vec<(SwPoint, Scalar)> = (0..n) + .into_par_iter() + .map(|i| { + let seed = ((n as u64) << 32) ^ (i as u64); + let s_bytes = bytes_from_seed(seed ^ 0x6a09e667f3bcc909); + let p_scalar = Scalar::from_bytes_mod_order(s_bytes); + let p_ed = base * p_scalar; + let point = SwPoint::from_edwards(&p_ed); + + let sc_bytes = bytes_from_seed(seed ^ 0xbb67ae8584caa73b); + let scalar = Scalar::from_bytes_mod_order(sc_bytes); + (point, scalar) + }) + .collect(); + + let mut points = Vec::with_capacity(n); + let mut points_ed = Vec::with_capacity(n); + let mut scalars = Vec::with_capacity(n); + for (point, scalar) in pairs { + points_ed.push(point.to_edwards().expect("valid sw point")); + points.push(point); + scalars.push(scalar); + } + + let expected_ed = edwards_msm(&points_ed, &scalars); + let expected = SwPoint::from_edwards(&expected_ed); + + let expected_bytes = sw_point_to_sppark_affine_bytes(&expected); + let out = msm_curve25519_gpu_bytes(&points, &scalars).expect("cuda msm"); + assert!( + affine_bytes_match(&out, &expected_bytes), + "unexpected MSM output" + ); + } +}