Merge pull request #4 from betrusted-io/cleanup-hw

Cleanup hw

.vscode/settings.json

@@ -1,5 +1,7 @@
 {
+  "rust-analyzer.cargo.target": "riscv32imac-unknown-xous-elf",
     "rust-analyzer.diagnostics.disabled": [
     "macro-error"
   ]
 }
+

curve25519-dalek/Cargo.toml

@@ -62,6 +62,8 @@ engine25519-as = {git = "https://github.com/betrusted-io/engine25519-as.git", re
 utralib = {version = "0.1.24", default-features = false}
 zeroize = { version = "1", default-features = false }
 xous = "0.9.58"
+# for fallback when hardware is unavailable
+fiat-crypto = { version = "0.2.1", default-features = false}
 
 [target.'cfg(target_arch = "x86_64")'.dependencies]
 cpufeatures = "0.2.6"
@@ -69,7 +71,9 @@ cpufeatures = "0.2.6"
 fiat-crypto = { version = "0.2.1", default-features = false , optional = true}
 
 [features]
-default = ["alloc", "precomputed-tables", "zeroize"]
+auto-release = []
+warn-fallback = []
+default = ["alloc", "precomputed-tables", "zeroize", "auto-release", "warn-fallback"]
 alloc = ["zeroize?/alloc"]
 precomputed-tables = []
 legacy_compatibility = []

curve25519-dalek/src/backend/serial/u32e/field.rs

@@ -24,8 +24,34 @@ use core::ops::{Sub, SubAssign};
 use subtle::Choice;
 use subtle::ConditionallySelectable;
 
+use core::fmt::Debug;
+use fiat_crypto::curve25519_32::*;
 use zeroize::Zeroize;
 
+#[derive(Copy, Clone)]
+pub struct FieldElement2625(pub(crate) fiat_25519_tight_field_element);
+
+impl Debug for FieldElement2625 {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        write!(f, "FieldElement2625({:?})", &(self.0).0[..])
+    }
+}
+
+#[cfg(feature = "zeroize")]
+impl Zeroize for FieldElement2625 {
+    fn zeroize(&mut self) {
+        (self.0).0.zeroize();
+    }
+}
+
+impl FieldElement2625 {
+    pub(crate) const fn from_limbs(limbs: [u32; 10]) -> FieldElement2625 {
+        FieldElement2625(fiat_25519_tight_field_element(limbs))
+    }
+
+    pub const ZERO: FieldElement2625 = FieldElement2625::from_limbs([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
+}
+
 /// A `Engine25519` represents an element of the field
 /// \\( \mathbb Z / (2\^{255} - 19)\\).
 ///
@@ -50,102 +76,180 @@ use zeroize::Zeroize;
 /// outside of the `curve25519_dalek::field` module.
 
 #[derive(Copy, Clone, Debug)]
-pub struct Engine25519(
+pub struct Engine25519(pub(crate) [u8; 32]);
-    pub (crate) [u8; 32]
-);
 pub(crate) enum EngineOp {
     Mul,
     Add,
     Sub,
 }
 
+pub fn bytes_to_fiat(data: &[u8; 32]) -> FieldElement2625 {
+    let mut temp = [0u8; 32];
+    temp.copy_from_slice(data);
+    temp[31] &= 127u8;
+    let mut output = fiat_25519_tight_field_element([0u32; 10]);
+    fiat_25519_from_bytes(&mut output, &temp);
+    FieldElement2625(output)
+}
+
+pub fn fiat_to_bytes(fiat_rep: &fiat_25519_tight_field_element) -> [u8; 32] {
+    let mut bytes = [0u8; 32];
+    fiat_25519_to_bytes(&mut bytes, fiat_rep);
+    bytes
+}
+
 #[allow(unused_qualifications)]
 pub(crate) fn engine(a: &[u8; 32], b: &[u8; 32], op: EngineOp) -> Engine25519 {
-    use utralib::generated::*;
     use crate::backend::serial::u32e::*;
 
-    crate::backend::serial::u32e::ensure_engine();
+    match crate::backend::serial::u32e::ensure_engine() {
-    let mut engine = utralib::CSR::new(unsafe{ENGINE_BASE.unwrap()}.as_mut_ptr() as *mut u32);
+        Ok(_) => {
-    let mcode: &'static mut [u32] = unsafe{
+            let mut engine =
-        core::slice::from_raw_parts_mut(ENGINE_MEM.unwrap().as_mut_ptr() as *mut u32, 1024)
+                utralib::CSR::new(unsafe { ENGINE_BASE.unwrap() }.as_mut_ptr() as *mut u32);
-    };
+            let mcode: &'static mut [u32] = unsafe {
-    let rf: [&'static mut [u32]; 3] = [
+                core::slice::from_raw_parts_mut(ENGINE_MEM.unwrap().as_mut_ptr() as *mut u32, 1024)
-        unsafe{core::slice::from_raw_parts_mut(
+            };
-            (ENGINE_MEM.unwrap().as_mut_ptr() as usize + 0x1_0000 + 0 * 32) as *mut u32, 8)},
+            let rf: [&'static mut [u32]; 3] = [
-        unsafe{core::slice::from_raw_parts_mut(
+                unsafe {
-            (ENGINE_MEM.unwrap().as_mut_ptr() as usize + 0x1_0000 + 1 * 32) as *mut u32, 8)},
+                    core::slice::from_raw_parts_mut(
-        unsafe{core::slice::from_raw_parts_mut(
+                        (ENGINE_MEM.unwrap().as_mut_ptr() as usize + 0x1_0000 + 0 * 32) as *mut u32,
-            (ENGINE_MEM.unwrap().as_mut_ptr() as usize + 0x1_0000 + 2 * 32) as *mut u32, 8)},
+                        8,
-    ];
+                    )
+                },
+                unsafe {
+                    core::slice::from_raw_parts_mut(
+                        (ENGINE_MEM.unwrap().as_mut_ptr() as usize + 0x1_0000 + 1 * 32) as *mut u32,
+                        8,
+                    )
+                },
+                unsafe {
+                    core::slice::from_raw_parts_mut(
+                        (ENGINE_MEM.unwrap().as_mut_ptr() as usize + 0x1_0000 + 2 * 32) as *mut u32,
+                        8,
+                    )
+                },
+            ];
+            loop {
+                let prog_len = match op {
+                    EngineOp::Mul => {
+                        let prog = assemble_engine25519!(
+                            start:
+                                mul %2, %0, %1
+                                fin
+                        );
+                        for (&src, dest) in prog.iter().zip(mcode.iter_mut()) {
+                            *dest = src;
+                        }
+                        engine.wfo(utra::engine::MPLEN_MPLEN, prog.len() as u32);
+                        prog.len()
+                    }
+                    EngineOp::Add => {
+                        let prog = assemble_engine25519!(
+                        start:
+                            add %2, %0, %1
+                            trd %30, %2
+                            sub %2, %2, %30
+                            fin
+                        );
+                        for (&src, dest) in prog.iter().zip(mcode.iter_mut()) {
+                            *dest = src;
+                        }
+                        engine.wfo(utra::engine::MPLEN_MPLEN, prog.len() as u32);
+                        prog.len()
+                    }
+                    EngineOp::Sub => {
+                        let prog = assemble_engine25519!(
+                        start:
+                            sub %1, #3, %1
+                            add %2, %0, %1
+                            trd %30, %2
+                            sub %2, %2, %30
+                            fin
+                        );
+                        for (&src, dest) in prog.iter().zip(mcode.iter_mut()) {
+                            *dest = src;
+                        }
+                        engine.wfo(utra::engine::MPLEN_MPLEN, prog.len() as u32);
+                        prog.len()
+                    }
+                };
+                // copy a arg
+                for (src, dst) in a.chunks_exact(4).zip(rf[0].iter_mut()) {
+                    let bytes: [u8; 4] = [src[0], src[1], src[2], src[3]];
+                    unsafe {
+                        (dst as *mut u32).write_volatile(u32::from_le_bytes(bytes));
+                    }
+                    /* this is a bad idea: src[0..4].try_into().unwrap()
+                    because "unwrap()" adds in a whole bunch of string formatting stuff, adds +16k or so to the binary size
+                    */
+                }
 
-    match op {
+                // copy b arg
-        EngineOp::Mul => {
+                for (src, dst) in b.chunks_exact(4).zip(rf[1].iter_mut()) {
-            let prog = assemble_engine25519!(
+                    let bytes: [u8; 4] = [src[0], src[1], src[2], src[3]];
-                start:
+                    unsafe {
-                    mul %2, %0, %1
+                        (dst as *mut u32).write_volatile(u32::from_le_bytes(bytes));
-                    fin
+                    }
-            );
+                }
-            for (&src, dest) in prog.iter().zip(mcode.iter_mut()) {
+
-                *dest = src;
+                engine.wfo(utra::engine::CONTROL_GO, 1);
+                while engine.rf(utra::engine::STATUS_RUNNING) != 0 {}
+                if !was_engine_error(prog_len) {
+                    break;
+                }
             }
-            engine.wfo(utra::engine::MPLEN_MPLEN, prog.len() as u32);
+
-        },
+            // return result, always in reg 2
-        EngineOp::Add => {
+            let mut result: [u8; 32] = [0; 32];
-            let prog = assemble_engine25519!(
+            for (&src, dst) in rf[2].iter().zip(result.chunks_exact_mut(4)) {
-            start:
+                for (&sb, db) in src.to_le_bytes().iter().zip(dst.iter_mut()) {
-                add %2, %0, %1
+                    *db = sb;
-                trd %30, %2
+                }
-                sub %2, %2, %30
-                fin
-            );
-            for (&src, dest) in prog.iter().zip(mcode.iter_mut()) {
-                *dest = src;
             }
-            engine.wfo(utra::engine::MPLEN_MPLEN, prog.len() as u32);
-        },
-        EngineOp::Sub => {
-            let prog = assemble_engine25519!(
-            start:
-                sub %1, #3, %1
-                add %2, %0, %1
-                trd %30, %2
-                sub %2, %2, %30
-                fin
-            );
-            for (&src, dest) in prog.iter().zip(mcode.iter_mut()) {
-                *dest = src;
-            }
-            engine.wfo(utra::engine::MPLEN_MPLEN, prog.len() as u32);
-        },
-    }
-    // copy a arg
-    for (src, dst) in a.chunks_exact(4).zip(rf[0].iter_mut()) {
-        let bytes: [u8; 4] = [src[0], src[1], src[2], src[3]];
-        unsafe{ (dst as *mut u32).write_volatile(u32::from_le_bytes(bytes));}
-        /* this is a bad idea: src[0..4].try_into().unwrap()
-           because "unwrap()" adds in a whole bunch of string formatting stuff, adds +16k or so to the binary size
-        */
-    }
 
-    // copy b arg
+            #[cfg(feature="auto-release")]
-    for (src, dst) in b.chunks_exact(4).zip(rf[1].iter_mut()) {
+            free_engine();
-        let bytes: [u8; 4] = [src[0], src[1], src[2], src[3]];
-        unsafe{ (dst as *mut u32).write_volatile(u32::from_le_bytes(bytes));}
-    }
 
-    engine.wfo(utra::engine::CONTROL_GO, 1);
+            Engine25519 { 0: result }
-    while engine.rf(utra::engine::STATUS_RUNNING) != 0 {}
+        }
-
+        _ => {
-    // return result, always in reg 2
+            // fallback to fiat crypto field arithmetic...
-    let mut result: [u8; 32] = [0; 32];
+            #[cfg(feature="warn-fallback")]
-    for (&src, dst) in rf[2].iter().zip(result.chunks_exact_mut(4)) {
+            log::warn!("Hardware acceleration unavailable, falling back to software");
-        for (&sb, db) in src.to_le_bytes().iter().zip(dst.iter_mut()) {
+            let fiat_a = bytes_to_fiat(a);
-            *db = sb;
+            let fiat_b = bytes_to_fiat(b);
+            match op {
+                EngineOp::Mul => {
+                    let mut self_loose = fiat_25519_loose_field_element([0; 10]);
+                    fiat_25519_relax(&mut self_loose, &fiat_a.0);
+                    let mut rhs_loose = fiat_25519_loose_field_element([0; 10]);
+                    fiat_25519_relax(&mut rhs_loose, &fiat_b.0);
+                    let mut output = FieldElement2625::ZERO;
+                    fiat_25519_carry_mul(&mut output.0, &self_loose, &rhs_loose);
+                    Engine25519 {
+                        0: fiat_to_bytes(&output.0),
+                    }
+                }
+                EngineOp::Add => {
+                    let mut result_loose = fiat_25519_loose_field_element([0; 10]);
+                    fiat_25519_add(&mut result_loose, &fiat_a.0, &fiat_b.0);
+                    let mut output = FieldElement2625::ZERO;
+                    fiat_25519_carry(&mut output.0, &result_loose);
+                    Engine25519 {
+                        0: fiat_to_bytes(&output.0),
+                    }
+                }
+                EngineOp::Sub => {
+                    let mut result_loose = fiat_25519_loose_field_element([0; 10]);
+                    fiat_25519_sub(&mut result_loose, &fiat_a.0, &fiat_b.0);
+                    let mut output = FieldElement2625::ZERO;
+                    fiat_25519_carry(&mut output.0, &result_loose);
+                    Engine25519 {
+                        0: fiat_to_bytes(&output.0),
+                    }
+                }
+            }
         }
-    }
-
-    Engine25519 {
-        0: result
     }
 }
 
@@ -210,11 +314,7 @@ impl<'a> Neg for &'a Engine25519 {
 }
 
 impl ConditionallySelectable for Engine25519 {
-    fn conditional_select(
+    fn conditional_select(a: &Engine25519, b: &Engine25519, choice: Choice) -> Engine25519 {
-        a: &Engine25519,
-        b: &Engine25519,
-        choice: Choice,
-    ) -> Engine25519 {
         Engine25519([
             u8::conditional_select(&a.0[0], &b.0[0], choice),
             u8::conditional_select(&a.0[1], &b.0[1], choice),
@@ -330,22 +430,23 @@ impl Engine25519 {
     }
 
     /// Construct zero.
-    pub const ZERO: Engine25519 = Engine25519([ 0 ; 32 ]);
+    pub const ZERO: Engine25519 = Engine25519([0; 32]);
 
     /// Construct one.
-    pub const ONE: Engine25519 = Engine25519([   1, 0, 0, 0, 0, 0, 0, 0,
+    pub const ONE: Engine25519 = Engine25519([
-                        0, 0, 0, 0, 0, 0, 0, 0,
+        1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-                        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0,
-                        0, 0, 0, 0, 0, 0, 0, 0,
+    ]);
-            ]);
 
     /// Construct -1.
-    pub const MINUS_ONE: Engine25519 =
+    pub const MINUS_ONE: Engine25519 = Engine25519([
-        Engine25519([236, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127]);
+        236, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+        255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127,
+    ]);
 
     /// Given `k > 0`, return `self^(2^k)`.
     pub fn pow2k(&self, k: u32) -> Engine25519 {
-        debug_assert!( k > 0 );
+        debug_assert!(k > 0);
         let mut z = self.square();
         for _ in 1..k {
             z = z.square();
@@ -364,12 +465,11 @@ impl Engine25519 {
     /// encoding of every field element should decode, re-encode to
     /// the canonical encoding, and check that the input was
     /// canonical.
-    pub fn from_bytes(data: &[u8; 32]) -> Engine25519 { //FeFromBytes
+    pub fn from_bytes(data: &[u8; 32]) -> Engine25519 {
+        //FeFromBytes
         let mut mask_data = data.clone();
         mask_data[31] &= 0x7F; // mask off the high bit per comment above
-        Engine25519 {
+        Engine25519 { 0: mask_data }
-            0: mask_data,
-        }
     }
 
     /// Serialize this `FieldElement51` to a 32-byte array.  The

curve25519-dalek/src/backend/serial/u32e/mod.rs

@@ -34,11 +34,12 @@ pub(crate) const RF_U8_BASE: usize = 0x1_0000;
 #[allow(dead_code)]
 pub(crate) const RF_U32_BASE: usize = 0x1_0000 / 4;
 
+/// It is safe to call this multiple times.
 pub fn free_engine() {
     log::debug!("free engine");
     if let Some(base) = unsafe { ENGINE_BASE.take() } {
         let mut engine = utralib::CSR::new(base.as_mut_ptr() as *mut u32);
-        engine.rmwf(utra::engine::POWER_ON, 1);
+        engine.rmwf(utra::engine::POWER_ON, 0);
         xous::unmap_memory(base).unwrap();
     }
     if let Some(mem) = unsafe { ENGINE_MEM.take() } {
@@ -46,15 +47,34 @@ pub fn free_engine() {
     }
 }
 
-pub fn ensure_engine() {
+/// Only safe to call this after ensure_engine() has been called.
+pub fn was_engine_error(job_len: usize) -> bool {
+    let mut engine = utralib::CSR::new(unsafe { ENGINE_BASE.unwrap() }.as_mut_ptr() as *mut u32);
+
+    let reason = engine.r(utra::engine::EV_PENDING);
+    if reason & engine.ms(utra::engine::EV_PENDING_ILLEGAL_OPCODE, 1) != 0 {
+        panic!("Illegal opcode encountered in engine25519");
+    }
+    // if the job length isn't what we had set it to, conclude that the
+    // microcode engine went through a suspend/resume cycle
+    if engine.rf(utra::engine::MPLEN_MPLEN) != job_len as u32 {
+        log::warn!("Suspend during engine25519 hw acceleration");
+        return true;
+    }
+
+    engine.wo(utra::engine::EV_PENDING, reason);
+    false
+}
+
+/// It is safe to call this multiple times.
+pub fn ensure_engine() -> Result<(), xous::Error> {
     if unsafe { ENGINE_BASE.is_none() } {
         let base = xous::syscall::map_memory(
             xous::MemoryAddress::new(utra::engine::HW_ENGINE_BASE),
             None,
             4096,
             xous::MemoryFlags::R | xous::MemoryFlags::W,
-        )
+        )?;
-        .expect("couldn't map engine CSR range");
         log::debug!("claiming engine csr {:x?}", base.as_ptr());
         unsafe {
             ENGINE_BASE = Some(base);
@@ -66,13 +86,14 @@ pub fn ensure_engine() {
             None,
             HW_ENGINE_MEM_LEN,
             xous::MemoryFlags::R | xous::MemoryFlags::W,
-        )
+        )?;
-        .expect("couldn't map engine memory window range");
         log::debug!("claiming engine mem {:x?}", mem.as_ptr());
         unsafe { ENGINE_MEM = Some(mem) };
     }
     let mut engine = utralib::CSR::new(unsafe { ENGINE_BASE.unwrap() }.as_mut_ptr() as *mut u32);
     engine.rmwf(utra::engine::POWER_ON, 1);
+    engine.wo(utra::engine::EV_PENDING, 0xFFFF_FFFF); // clear all pending bits
+    Ok(())
 }
 
 /// Safety: must be called after ensure_engine()

curve25519-dalek/src/field.rs

@@ -43,7 +43,7 @@ cfg_if! {
         ///
         /// The `FieldElement` type is an alias for one of the platform-specific
         /// implementations.
-        pub type FieldElement = backend::serial::u32e::field::Engine25519; 
+        pub type FieldElement = Engine25519;
 
     } else if #[cfg(curve25519_dalek_backend = "fiat")] {
         /// A `FieldElement` represents an element of the field

curve25519-dalek/src/montgomery.rs

@@ -54,10 +54,7 @@ use core::{
     ops::{Mul, MulAssign},
 };
 
-#[cfg(not(curve25519_dalek_backend = "u32e_backend"))]
 use crate::constants::{APLUS2_OVER_FOUR, MONTGOMERY_A, MONTGOMERY_A_NEG};
-#[cfg(curve25519_dalek_backend = "u32e_backend")]
-use crate::constants::{MONTGOMERY_A, MONTGOMERY_A_NEG}; // eliminate constants absorbed into the microcode engine
 
 use crate::edwards::{CompressedEdwardsY, EdwardsPoint};
 use crate::field::FieldElement;
@@ -465,15 +462,33 @@ impl ProjectivePoint {
         );
 
         use crate::backend::serial::u32e::*;
-        ensure_engine();
+        match ensure_engine() {
-        // safety: these were called after ensure_engine()
+            Ok(_) => {
-        let mut ucode_hw = unsafe { get_ucode() };
+                // safety: these were called after ensure_engine()
-        let rf_hw = unsafe { get_rf() };
+                let mut ucode_hw = unsafe { get_ucode() };
+                let rf_hw = unsafe { get_rf() };
 
-        copy_to_rf(self.U.as_bytes(), 29, rf_hw, 0);
+                let mut r;
-        copy_to_rf(self.W.as_bytes(), 30, rf_hw, 0);
+                loop {
+                    copy_to_rf(self.U.as_bytes(), 29, rf_hw, 0);
+                    copy_to_rf(self.W.as_bytes(), 30, rf_hw, 0);
 
-        MontgomeryPoint(run_job(&mut ucode_hw, &rf_hw, &mcode, 0))
+                    r = MontgomeryPoint(run_job(&mut ucode_hw, &rf_hw, &mcode, 0));
+                    if !was_engine_error(mcode.len()) {
+                        break;
+                    }
+                }
+                #[cfg(feature="auto-release")]
+                free_engine();
+                r
+            }
+            _ => {
+                #[cfg(feature="warn-fallback")]
+                log::warn!("Hardware acceleration unavailable, falling back to software");
+                let u = &self.U * &self.W.invert();
+                MontgomeryPoint(u.as_bytes())
+            }
+        }
     }
 }
 
@@ -622,29 +637,76 @@ pub(crate) fn differential_add_and_double(
             fin  // finish execution
     );
     use crate::backend::serial::u32e::*;
-    ensure_engine();
+    match ensure_engine() {
-    // safety: these were called after ensure_engine()
+        Ok(_) => {
-    let mut ucode_hw = unsafe { get_ucode() };
+            // safety: these were called after ensure_engine()
-    let rf_hw = unsafe { get_rf() };
+            let mut ucode_hw = unsafe { get_ucode() };
+            let rf_hw = unsafe { get_rf() };
 
-    // P.U in %20
+            loop {
-    // P.W in %21
+                // P.U in %20
-    // Q.U in %22
+                // P.W in %21
-    // Q.W in %23
+                // Q.U in %22
-    // affine_PmQ in %24
+                // Q.W in %23
-    copy_to_rf(P.U.as_bytes(), 20, rf_hw, 0);
+                // affine_PmQ in %24
-    copy_to_rf(P.W.as_bytes(), 21, rf_hw, 0);
+                copy_to_rf(P.U.as_bytes(), 20, rf_hw, 0);
-    copy_to_rf(Q.U.as_bytes(), 22, rf_hw, 0);
+                copy_to_rf(P.W.as_bytes(), 21, rf_hw, 0);
-    copy_to_rf(Q.W.as_bytes(), 23, rf_hw, 0);
+                copy_to_rf(Q.U.as_bytes(), 22, rf_hw, 0);
-    copy_to_rf(affine_PmQ.as_bytes(), 24, rf_hw, 0);
+                copy_to_rf(Q.W.as_bytes(), 23, rf_hw, 0);
+                copy_to_rf(affine_PmQ.as_bytes(), 24, rf_hw, 0);
 
-    // start the run
+                // start the run
-    run_job(&mut ucode_hw, &rf_hw, &mcode, 0);
+                run_job(&mut ucode_hw, &rf_hw, &mcode, 0);
+                if !was_engine_error(mcode.len()) {
+                    break;
+                }
+            }
 
-    P.U = FieldElement::from_bytes(&copy_from_rf(20, &rf_hw, 0));
+            P.U = FieldElement::from_bytes(&copy_from_rf(20, &rf_hw, 0));
-    P.W = FieldElement::from_bytes(&copy_from_rf(21, &rf_hw, 0));
+            P.W = FieldElement::from_bytes(&copy_from_rf(21, &rf_hw, 0));
-    Q.U = FieldElement::from_bytes(&copy_from_rf(22, &rf_hw, 0));
+            Q.U = FieldElement::from_bytes(&copy_from_rf(22, &rf_hw, 0));
-    Q.W = FieldElement::from_bytes(&copy_from_rf(23, &rf_hw, 0));
+            Q.W = FieldElement::from_bytes(&copy_from_rf(23, &rf_hw, 0));
+            #[cfg(feature="auto-release")]
+            free_engine();
+        }
+        _ => {
+            #[cfg(feature="warn-fallback")]
+            log::warn!("Hardware acceleration unavailable, falling back to software");
+            let t0 = &P.U + &P.W;
+            let t1 = &P.U - &P.W;
+            let t2 = &Q.U + &Q.W;
+            let t3 = &Q.U - &Q.W;
+
+            let t4 = t0.square();   // (U_P + W_P)^2 = U_P^2 + 2 U_P W_P + W_P^2
+            let t5 = t1.square();   // (U_P - W_P)^2 = U_P^2 - 2 U_P W_P + W_P^2
+
+            let t6 = &t4 - &t5;     // 4 U_P W_P
+
+            let t7 = &t0 * &t3;     // (U_P + W_P) (U_Q - W_Q) = U_P U_Q + W_P U_Q - U_P W_Q - W_P W_Q
+            let t8 = &t1 * &t2;     // (U_P - W_P) (U_Q + W_Q) = U_P U_Q - W_P U_Q + U_P W_Q - W_P W_Q
+
+            let t9  = &t7 + &t8;    // 2 (U_P U_Q - W_P W_Q)
+            let t10 = &t7 - &t8;    // 2 (W_P U_Q - U_P W_Q)
+
+            let t11 =  t9.square(); // 4 (U_P U_Q - W_P W_Q)^2
+            let t12 = t10.square(); // 4 (W_P U_Q - U_P W_Q)^2
+
+            let t13 = &APLUS2_OVER_FOUR * &t6; // (A + 2) U_P U_Q
+
+            let t14 = &t4 * &t5;    // ((U_P + W_P)(U_P - W_P))^2 = (U_P^2 - W_P^2)^2
+            let t15 = &t13 + &t5;   // (U_P - W_P)^2 + (A + 2) U_P W_P
+
+            let t16 = &t6 * &t15;   // 4 (U_P W_P) ((U_P - W_P)^2 + (A + 2) U_P W_P)
+
+            let t17 = affine_PmQ * &t12; // U_D * 4 (W_P U_Q - U_P W_Q)^2
+            let t18 = t11;               // W_D * 4 (U_P U_Q - W_P W_Q)^2
+
+            P.U = t14;  // U_{P'} = (U_P + W_P)^2 (U_P - W_P)^2
+            P.W = t16;  // W_{P'} = (4 U_P W_P) ((U_P - W_P)^2 + ((A + 2)/4) 4 U_P W_P)
+            Q.U = t18;  // U_{Q'} = W_D * 4 (U_P U_Q - W_P W_Q)^2
+            Q.W = t17;  // W_{Q'} = U_D * 4 (W_P U_Q - U_P W_Q)^2
+        }
+    }
 }
 
 define_mul_assign_variants!(LHS = MontgomeryPoint, RHS = Scalar);
@@ -945,35 +1007,53 @@ impl Mul<&Scalar> for &MontgomeryPoint {
         );
 
         let window = 0;
-        ensure_engine();
+        match ensure_engine() {
-        // safety: these were called after ensure_engine()
+            Ok(_) => {
-        let mut ucode_hw = unsafe { get_ucode() };
+                let mut r;
-        let mut rf_hw = unsafe { get_rf() };
+                loop {
+                    // safety: these were called after ensure_engine()
+                    let mut ucode_hw = unsafe { get_ucode() };
+                    let mut rf_hw = unsafe { get_rf() };
 
-        copy_to_rf(x0.U.as_bytes(), 25, &mut rf_hw, window);
+                    copy_to_rf(x0.U.as_bytes(), 25, &mut rf_hw, window);
-        copy_to_rf(x0.W.as_bytes(), 26, &mut rf_hw, window);
+                    copy_to_rf(x0.W.as_bytes(), 26, &mut rf_hw, window);
-        copy_to_rf(x1.U.as_bytes(), 27, &mut rf_hw, window);
+                    copy_to_rf(x1.U.as_bytes(), 27, &mut rf_hw, window);
-        copy_to_rf(x1.W.as_bytes(), 28, &mut rf_hw, window);
+                    copy_to_rf(x1.W.as_bytes(), 28, &mut rf_hw, window);
-        copy_to_rf(affine_u.as_bytes(), 24, &mut rf_hw, window);
+                    copy_to_rf(affine_u.as_bytes(), 24, &mut rf_hw, window);
-        copy_to_rf(scalar.bytes, 31, &mut rf_hw, window);
+                    copy_to_rf(scalar.bytes, 31, &mut rf_hw, window);
-        copy_to_rf(
+                    copy_to_rf(
-            [
+                        [
-                254, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                            254, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-                0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-                0x00, 0x00, 0x00, 0x00,
+                            0x00, 0x00, 0x00, 0x00,
-            ],
+                        ],
-            19,
+                        19,
-            &mut rf_hw,
+                        &mut rf_hw,
-            window,
+                        window,
-        ); // 254 as loop counter
+                    ); // 254 as loop counter
 
-        MontgomeryPoint(run_job(&mut ucode_hw, &rf_hw, &mcode, window))
+                    r = MontgomeryPoint(run_job(&mut ucode_hw, &rf_hw, &mcode, window));
+                    if !was_engine_error(mcode.len()) {
+                        break;
+                    }
+                }
+                #[cfg(feature="auto-release")]
+                free_engine();
+                r
+            }
+            _ => {
+                #[cfg(feature="warn-fallback")]
+                log::warn!("Hardware acceleration unavailable, falling back to software");
+                // We multiply by the integer representation of the given Scalar. By scalar invariant #1,
+                // the MSB is 0, so we can skip it.
+                self.mul_bits_be(scalar.bits_le().rev().skip(1))
+            }
+        }
     }
 
     /// Given `self` \\( = u\_0(P) \\), and a `Scalar` \\(n\\), return \\( u\_0(\[n\]P) \\)
     #[cfg(not(curve25519_dalek_backend = "u32e_backend"))]
     fn mul(self, scalar: &Scalar) -> MontgomeryPoint {
-        // TODO: consider feature "panic_on_sw_eval"
         #[cfg(all(not(test), curve25519_dalek_backend = "u32e_backend"))] // due to issue https://github.com/rust-lang/rust/issues/59168, you will have to manually comment this out when running a test on the full system and not just this crate.
         log::warn!("sw montgomery multiply being used - check for build config errors!");
         // We multiply by the integer representation of the given Scalar. By scalar invariant #1,

curve25519-dalek/src/scalar.rs

@@ -844,7 +844,6 @@ impl Scalar {
     }
 
     /// Get the bits of the scalar, in little-endian order
-    #[cfg(not(curve25519_dalek_backend = "u32e_backend"))]
     pub(crate) fn bits_le(&self) -> impl DoubleEndedIterator<Item = bool> + '_ {
         (0..256).map(|i| {
             // As i runs from 0..256, the bottom 3 bits index the bit, while the upper bits index