better error handling for batch verify (#57)

curve25519/solana-ed25519/src/ed_sigs/batch.rs

@@ -182,7 +182,7 @@ impl Verifier {
         for (vk_bytes, sigs) in self.signatures.iter() {
             let A = CompressedEdwardsY(vk_bytes.0)
                 .decompress()
-                .ok_or(Error::InvalidSignature)?;
+                .ok_or(Error::MalformedPublicKey)?;
 
             let mut A_coeff = Scalar::ZERO;
 

curve25519/solana-ed25519/src/ed_sigs/tests/batch.rs

@@ -1,6 +1,7 @@
 #![cfg(all(feature = "alloc", feature = "rand_core"))]
 
 use crate::ed_sigs::*;
+use crate::edwards::CompressedEdwardsY;
 use alloc::vec::Vec;
 
 #[test]
@@ -47,3 +48,37 @@ fn batch_verify_with_one_bad_sig() {
         }
     }
 }
+
+#[test]
+fn batch_verify_with_malformed_verification_key() {
+    let seed = [1u8; 32];
+    let sk = SigningKey::from(seed);
+    let msg = b"BatchVerifyTest";
+    let sig = sk.sign(&msg[..]);
+    let malformed_key = VerificationKeyBytes::from(first_undecodable_compressed_edwards_y());
+
+    assert_eq!(
+        VerificationKey::try_from(malformed_key),
+        Err(Error::MalformedPublicKey)
+    );
+
+    let mut batch = batch::Verifier::new();
+    batch.queue((malformed_key, sig, msg));
+
+    assert_eq!(
+        batch.verify(rand::thread_rng()),
+        Err(Error::MalformedPublicKey)
+    );
+}
+
+fn first_undecodable_compressed_edwards_y() -> [u8; 32] {
+    for candidate in 0u16..=u16::MAX {
+        let mut bytes = [0u8; 32];
+        bytes[..2].copy_from_slice(&candidate.to_le_bytes());
+        if CompressedEdwardsY(bytes).decompress().is_none() {
+            return bytes;
+        }
+    }
+
+    panic!("failed to find an undecodable compressed Edwards-Y encoding");
+}