From 0ef7c1fa19f7fe2a67392301457ba30bd6e7352a Mon Sep 17 00:00:00 2001 From: Paul Kehrer Date: Sat, 1 Feb 2025 17:26:59 -0800 Subject: [PATCH] add XOFHash (#12380) * add XOFHash * refactors for comments * use cfg_if * fix docs, fix linting * don't expose squeeze on unsupported things * smaller strides * ellipsis --- .github/workflows/ci.yml | 2 +- CHANGELOG.rst | 3 + .../primitives/cryptographic-hashes.rst | 149 ++++++++++++++---- .../bindings/_rust/openssl/__init__.pyi | 1 + .../hazmat/bindings/_rust/openssl/hashes.pyi | 8 + src/cryptography/hazmat/primitives/hashes.py | 3 + src/rust/Cargo.toml | 2 +- src/rust/build.rs | 3 + src/rust/src/backend/hashes.rs | 109 ++++++++++++- src/rust/src/lib.rs | 4 + tests/hazmat/primitives/test_xofhash.py | 131 +++++++++++++++ 11 files changed, 377 insertions(+), 38 deletions(-) create mode 100644 tests/hazmat/primitives/test_xofhash.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72b921657..ce1efbff0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: PYTHON: - {VERSION: "3.12", NOXSESSION: "flake"} - {VERSION: "3.12", NOXSESSION: "rust"} - - {VERSION: "3.12", NOXSESSION: "docs", OPENSSL: {TYPE: "openssl", VERSION: "3.2.3"}} + - {VERSION: "3.12", NOXSESSION: "docs", OPENSSL: {TYPE: "openssl", VERSION: "3.4.0"}} - {VERSION: "3.13", NOXSESSION: "tests"} - {VERSION: "3.14-dev", NOXSESSION: "tests"} - {VERSION: "pypy-3.10", NOXSESSION: "tests-nocoverage"} diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3461f0a43..5075336f9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -21,6 +21,9 @@ Changelog was raised). * Added ``unsafe_skip_rsa_key_validation`` keyword-argument to :func:`~cryptography.hazmat.primitives.serialization.load_ssh_private_key`. +* Added :class:`~cryptography.hazmat.primitives.hashes.XOFHash` to support + repeated :meth:`~cryptography.hazmat.primitives.hashes.XOFHash.squeeze` + operations on extendable output functions. .. _v44-0-0: diff --git a/docs/hazmat/primitives/cryptographic-hashes.rst b/docs/hazmat/primitives/cryptographic-hashes.rst index c1c29cad2..1bf8ccf81 100644 --- a/docs/hazmat/primitives/cryptographic-hashes.rst +++ b/docs/hazmat/primitives/cryptographic-hashes.rst @@ -35,7 +35,7 @@ Message digests (Hashing) :param algorithm: A :class:`~cryptography.hazmat.primitives.hashes.HashAlgorithm` - instance such as those described in + instance such as those described :ref:`below `. :raises cryptography.exceptions.UnsupportedAlgorithm: This is raised if the @@ -44,14 +44,14 @@ Message digests (Hashing) .. method:: update(data) :param bytes data: The bytes to be hashed. - :raises cryptography.exceptions.AlreadyFinalized: See :meth:`finalize`. + :raises cryptography.exceptions.AlreadyFinalized: See :meth:`.finalize`. :raises TypeError: This exception is raised if ``data`` is not ``bytes``. .. method:: copy() Copy this :class:`Hash` instance, usually so that you may call - :meth:`finalize` to get an intermediate digest value while we continue - to call :meth:`update` on the original instance. + :meth:`.finalize` to get an intermediate digest value while we continue + to call :meth:`.update` on the original instance. :return: A new instance of :class:`Hash` that can be updated and finalized independently of the original instance. @@ -62,11 +62,70 @@ Message digests (Hashing) Finalize the current context and return the message digest as bytes. After ``finalize`` has been called this object can no longer be used - and :meth:`update`, :meth:`copy`, and :meth:`finalize` will raise an + and :meth:`.update`, :meth:`.copy`, and :meth:`.finalize` will raise an :class:`~cryptography.exceptions.AlreadyFinalized` exception. :return bytes: The message digest as bytes. +.. class:: XOFHash(algorithm) + + An extendable output function (XOF) is a cryptographic hash function that + can produce an arbitrary amount of output for a given input. The output + can be obtained by repeatedly calling :meth:`.squeeze` with the desired + length. + + .. doctest:: + + >>> import sys + >>> from cryptography.hazmat.primitives import hashes + >>> digest = hashes.XOFHash(hashes.SHAKE128(digest_size=sys.maxsize)) + >>> digest.update(b"abc") + >>> digest.update(b"123") + >>> digest.squeeze(16) + b'\x18\xd6\xbd\xeb5u\x83[@\xfa%/\xdc\xca\x9f\x1b' + >>> digest.squeeze(16) + b'\xc2\xeb\x12\x05\xc3\xf9Bu\x88\xe0\xda\x80FvAV' + + :param algorithm: A + :class:`~cryptography.hazmat.primitives.hashes.ExtendableOutputFunction` + instance such as those described + :ref:`below `. The ``digest_size`` + passed is the maximum number of bytes that can be squeezed from the XOF + when using this class. + + :raises cryptography.exceptions.UnsupportedAlgorithm: This is raised if the + provided ``algorithm`` is unsupported. + + .. method:: update(data) + + :param bytes data: The bytes to be hashed. + :raises cryptography.exceptions.AlreadyFinalized: If already squeezed. + :raises TypeError: This exception is raised if ``data`` is not ``bytes``. + + .. method:: copy() + + Copy this :class:`XOFHash` instance, usually so that you may call + :meth:`.squeeze` to get an intermediate digest value while we continue + to call :meth:`.update` on the original instance. + + :return: A new instance of :class:`XOFHash` that can be updated + and squeezed independently of the original instance. If + you copy an instance that has already been squeezed, the copy will + also be in a squeezed state. + :raises cryptography.exceptions.AlreadyFinalized: See :meth:`.squeeze`. + + .. method:: squeeze(length) + + :param int length: The number of bytes to squeeze. + + After :meth:`.squeeze` has been called this object can no longer be updated + and :meth:`.update`, will raise an + :class:`~cryptography.exceptions.AlreadyFinalized` exception. + + :return bytes: ``length`` bytes of output from the extendable output function (XOF). + :raises ValueError: If the maximum number of bytes that can be squeezed + has been exceeded. + .. _cryptographic-hash-algorithms: @@ -176,36 +235,6 @@ than SHA-2 so at this time most users should choose SHA-2. SHA3/512 is a cryptographic hash function from the SHA-3 family and is standardized by NIST. It produces a 512-bit message digest. -.. class:: SHAKE128(digest_size) - - .. versionadded:: 2.5 - - SHAKE128 is an extendable output function (XOF) based on the same core - permutations as SHA3. It allows the caller to obtain an arbitrarily long - digest length. Longer lengths, however, do not increase security or - collision resistance and lengths shorter than 128 bit (16 bytes) will - decrease it. - - :param int digest_size: The length of output desired. Must be greater than - zero. - - :raises ValueError: If the ``digest_size`` is invalid. - -.. class:: SHAKE256(digest_size) - - .. versionadded:: 2.5 - - SHAKE256 is an extendable output function (XOF) based on the same core - permutations as SHA3. It allows the caller to obtain an arbitrarily long - digest length. Longer lengths, however, do not increase security or - collision resistance and lengths shorter than 256 bit (32 bytes) will - decrease it. - - :param int digest_size: The length of output desired. Must be greater than - zero. - - :raises ValueError: If the ``digest_size`` is invalid. - SHA-1 ~~~~~ @@ -250,6 +279,52 @@ SM3 `draft-sca-cfrg-sm3`_.) This hash should be used for compatibility purposes where required and is not otherwise recommended for use. +.. _extendable-output-functions: + +Extendable Output Functions +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. class:: SHAKE128(digest_size) + + .. versionadded:: 2.5 + + SHAKE128 is an extendable output function (XOF) based on the same core + permutations as SHA3. It allows the caller to obtain an arbitrarily long + digest length. Longer lengths, however, do not increase security or + collision resistance and lengths shorter than 128 bit (16 bytes) will + decrease it. + + This class can be used with :class:`Hash` or :class:`XOFHash`. When used + in :class:`Hash` :meth:`~cryptography.hazmat.primitives.hashes.Hash.finalize` + will return ``digest_size`` bytes. When used in :class:`XOFHash` this + defines the total number of bytes allowed to be squeezed. + + :param int digest_size: The length of output desired. Must be greater than + zero. + + :raises ValueError: If the ``digest_size`` is invalid. + +.. class:: SHAKE256(digest_size) + + .. versionadded:: 2.5 + + SHAKE256 is an extendable output function (XOF) based on the same core + permutations as SHA3. It allows the caller to obtain an arbitrarily long + digest length. Longer lengths, however, do not increase security or + collision resistance and lengths shorter than 256 bit (32 bytes) will + decrease it. + + This class can be used with :class:`Hash` or :class:`XOFHash`. When used + in :class:`Hash` :meth:`~cryptography.hazmat.primitives.hashes.Hash.finalize` + will return ``digest_size`` bytes. When used in :class:`XOFHash` this + defines the total number of bytes allowed to be squeezed. + + :param int digest_size: The length of output desired. Must be greater than + zero. + + :raises ValueError: If the ``digest_size`` is invalid. + + Interfaces ~~~~~~~~~~ @@ -269,6 +344,10 @@ Interfaces The size of the resulting digest in bytes. +.. class:: ExtendableOutputFunction + + An interface applied to hashes that act as extendable output functions (XOFs). + The currently supported XOFs are :class:`SHAKE128` and :class:`SHAKE256`. .. class:: HashContext diff --git a/src/cryptography/hazmat/bindings/_rust/openssl/__init__.pyi b/src/cryptography/hazmat/bindings/_rust/openssl/__init__.pyi index 600b48d7d..31baefd6d 100644 --- a/src/cryptography/hazmat/bindings/_rust/openssl/__init__.pyi +++ b/src/cryptography/hazmat/bindings/_rust/openssl/__init__.pyi @@ -50,6 +50,7 @@ CRYPTOGRAPHY_IS_BORINGSSL: bool CRYPTOGRAPHY_OPENSSL_300_OR_GREATER: bool CRYPTOGRAPHY_OPENSSL_309_OR_GREATER: bool CRYPTOGRAPHY_OPENSSL_320_OR_GREATER: bool +CRYPTOGRAPHY_OPENSSL_330_OR_GREATER: bool CRYPTOGRAPHY_OPENSSL_350_OR_GREATER: bool class Providers: ... diff --git a/src/cryptography/hazmat/bindings/_rust/openssl/hashes.pyi b/src/cryptography/hazmat/bindings/_rust/openssl/hashes.pyi index 56f317001..c6a853d7b 100644 --- a/src/cryptography/hazmat/bindings/_rust/openssl/hashes.pyi +++ b/src/cryptography/hazmat/bindings/_rust/openssl/hashes.pyi @@ -17,3 +17,11 @@ class Hash(hashes.HashContext): def copy(self) -> Hash: ... def hash_supported(algorithm: hashes.HashAlgorithm) -> bool: ... + +class XOFHash: + def __init__(self, algorithm: hashes.ExtendableOutputFunction) -> None: ... + @property + def algorithm(self) -> hashes.ExtendableOutputFunction: ... + def update(self, data: bytes) -> None: ... + def squeeze(self, length: int) -> bytes: ... + def copy(self) -> XOFHash: ... diff --git a/src/cryptography/hazmat/primitives/hashes.py b/src/cryptography/hazmat/primitives/hashes.py index b819e3992..77a5d273f 100644 --- a/src/cryptography/hazmat/primitives/hashes.py +++ b/src/cryptography/hazmat/primitives/hashes.py @@ -30,6 +30,7 @@ __all__ = [ "Hash", "HashAlgorithm", "HashContext", + "XOFHash", ] @@ -87,6 +88,8 @@ class HashContext(metaclass=abc.ABCMeta): Hash = rust_openssl.hashes.Hash HashContext.register(Hash) +XOFHash = rust_openssl.hashes.XOFHash + class ExtendableOutputFunction(metaclass=abc.ABCMeta): """ diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml index f9aa95788..8839f72bc 100644 --- a/src/rust/Cargo.toml +++ b/src/rust/Cargo.toml @@ -33,4 +33,4 @@ name = "cryptography_rust" crate-type = ["cdylib"] [lints.rust] -unexpected_cfgs = { level = "warn", check-cfg = ['cfg(CRYPTOGRAPHY_OPENSSL_300_OR_GREATER)', 'cfg(CRYPTOGRAPHY_OPENSSL_309_OR_GREATER)', 'cfg(CRYPTOGRAPHY_OPENSSL_320_OR_GREATER)', 'cfg(CRYPTOGRAPHY_OPENSSL_350_OR_GREATER)', 'cfg(CRYPTOGRAPHY_IS_LIBRESSL)', 'cfg(CRYPTOGRAPHY_IS_BORINGSSL)', 'cfg(CRYPTOGRAPHY_OSSLCONF, values("OPENSSL_NO_IDEA", "OPENSSL_NO_CAST", "OPENSSL_NO_BF", "OPENSSL_NO_CAMELLIA", "OPENSSL_NO_SEED", "OPENSSL_NO_SM4"))'] } +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(CRYPTOGRAPHY_OPENSSL_300_OR_GREATER)', 'cfg(CRYPTOGRAPHY_OPENSSL_309_OR_GREATER)', 'cfg(CRYPTOGRAPHY_OPENSSL_320_OR_GREATER)', 'cfg(CRYPTOGRAPHY_OPENSSL_330_OR_GREATER)', 'cfg(CRYPTOGRAPHY_OPENSSL_350_OR_GREATER)', 'cfg(CRYPTOGRAPHY_IS_LIBRESSL)', 'cfg(CRYPTOGRAPHY_IS_BORINGSSL)', 'cfg(CRYPTOGRAPHY_OSSLCONF, values("OPENSSL_NO_IDEA", "OPENSSL_NO_CAST", "OPENSSL_NO_BF", "OPENSSL_NO_CAMELLIA", "OPENSSL_NO_SEED", "OPENSSL_NO_SM4"))'] } diff --git a/src/rust/build.rs b/src/rust/build.rs index b6d750c6d..6edc4d524 100644 --- a/src/rust/build.rs +++ b/src/rust/build.rs @@ -18,6 +18,9 @@ fn main() { if version >= 0x3_02_00_00_0 { println!("cargo:rustc-cfg=CRYPTOGRAPHY_OPENSSL_320_OR_GREATER"); } + if version >= 0x3_03_00_00_0 { + println!("cargo:rustc-cfg=CRYPTOGRAPHY_OPENSSL_330_OR_GREATER"); + } if version >= 0x3_05_00_00_0 { println!("cargo:rustc-cfg=CRYPTOGRAPHY_OPENSSL_350_OR_GREATER"); } diff --git a/src/rust/src/backend/hashes.rs b/src/rust/src/backend/hashes.rs index 2d917ab82..4eec658a6 100644 --- a/src/rust/src/backend/hashes.rs +++ b/src/rust/src/backend/hashes.rs @@ -137,8 +137,115 @@ impl Hash { } } +#[pyo3::pyclass(module = "cryptography.hazmat.bindings._rust.openssl.hashes")] +pub(crate) struct XOFHash { + #[pyo3(get)] + algorithm: pyo3::Py, + ctx: openssl::hash::Hasher, + bytes_remaining: u64, + squeezed: bool, +} + +impl XOFHash { + pub(crate) fn update_bytes(&mut self, data: &[u8]) -> CryptographyResult<()> { + self.ctx.update(data)?; + Ok(()) + } +} + +#[pyo3::pymethods] +impl XOFHash { + #[new] + #[pyo3(signature = (algorithm))] + fn new( + py: pyo3::Python<'_>, + algorithm: &pyo3::Bound<'_, pyo3::PyAny>, + ) -> CryptographyResult { + cfg_if::cfg_if! { + if #[cfg(any( + CRYPTOGRAPHY_IS_LIBRESSL, + CRYPTOGRAPHY_IS_BORINGSSL, + not(CRYPTOGRAPHY_OPENSSL_330_OR_GREATER) + ))] { + let _ = py; + let _ = algorithm; + Err(CryptographyError::from( + exceptions::UnsupportedAlgorithm::new_err(( + "Extendable output functions are not supported on LibreSSL or BoringSSL.", + )), + )) + } else { + if !algorithm.is_instance(&types::EXTENDABLE_OUTPUT_FUNCTION.get(py)?)? { + return Err(CryptographyError::from( + pyo3::exceptions::PyTypeError::new_err( + "Expected instance of an extendable output function.", + ), + )); + } + let md = message_digest_from_algorithm(py, algorithm)?; + let ctx = openssl::hash::Hasher::new(md)?; + // We treat digest_size as the maximum total output for this API + let bytes_remaining = algorithm + .getattr(pyo3::intern!(py, "digest_size"))? + .extract::()?; + + Ok(XOFHash { + algorithm: algorithm.clone().unbind(), + ctx, + bytes_remaining, + squeezed: false, + }) + } + } + } + + fn update(&mut self, data: CffiBuf<'_>) -> CryptographyResult<()> { + if self.squeezed { + return Err(CryptographyError::from( + exceptions::AlreadyFinalized::new_err("Context was already squeezed."), + )); + } + self.update_bytes(data.as_bytes()) + } + #[cfg(all( + CRYPTOGRAPHY_OPENSSL_330_OR_GREATER, + not(CRYPTOGRAPHY_IS_LIBRESSL), + not(CRYPTOGRAPHY_IS_BORINGSSL), + ))] + fn squeeze<'p>( + &mut self, + py: pyo3::Python<'p>, + length: usize, + ) -> CryptographyResult> { + self.squeezed = true; + // We treat digest_size as the maximum total output for this API + self.bytes_remaining = self + .bytes_remaining + .checked_sub(length.try_into().unwrap()) + .ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err( + "Exceeded maximum squeeze limit specified by digest_size.", + ) + })?; + let result = pyo3::types::PyBytes::new_with(py, length, |b| { + self.ctx.squeeze_xof(b).unwrap(); + Ok(()) + })?; + Ok(result) + } + + fn copy(&self, py: pyo3::Python<'_>) -> CryptographyResult { + Ok(XOFHash { + algorithm: self.algorithm.clone_ref(py), + ctx: self.ctx.clone(), + bytes_remaining: self.bytes_remaining, + squeezed: self.squeezed, + }) + } +} + #[pyo3::pymodule] pub(crate) mod hashes { #[pymodule_export] - use super::{hash_supported, Hash}; + use super::{hash_supported, Hash, XOFHash}; } diff --git a/src/rust/src/lib.rs b/src/rust/src/lib.rs index cd3b9c11a..cd05334bf 100644 --- a/src/rust/src/lib.rs +++ b/src/rust/src/lib.rs @@ -210,6 +210,10 @@ mod _rust { "CRYPTOGRAPHY_OPENSSL_320_OR_GREATER", cfg!(CRYPTOGRAPHY_OPENSSL_320_OR_GREATER), )?; + openssl_mod.add( + "CRYPTOGRAPHY_OPENSSL_330_OR_GREATER", + cfg!(CRYPTOGRAPHY_OPENSSL_330_OR_GREATER), + )?; openssl_mod.add( "CRYPTOGRAPHY_OPENSSL_350_OR_GREATER", cfg!(CRYPTOGRAPHY_OPENSSL_350_OR_GREATER), diff --git a/tests/hazmat/primitives/test_xofhash.py b/tests/hazmat/primitives/test_xofhash.py new file mode 100644 index 000000000..ae9e88cd8 --- /dev/null +++ b/tests/hazmat/primitives/test_xofhash.py @@ -0,0 +1,131 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + + +import binascii +import os +import random +import sys + +import pytest + +from cryptography.exceptions import AlreadyFinalized, UnsupportedAlgorithm +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import hashes + +from ...utils import load_nist_vectors +from .utils import _load_all_params + + +@pytest.mark.supported( + only_if=lambda backend: ( + not rust_openssl.CRYPTOGRAPHY_OPENSSL_330_OR_GREATER + or rust_openssl.CRYPTOGRAPHY_IS_LIBRESSL + or rust_openssl.CRYPTOGRAPHY_IS_BORINGSSL + ), + skip_message="Requires backend without XOF support", +) +def test_unsupported_boring_libre(backend): + with pytest.raises(UnsupportedAlgorithm): + hashes.XOFHash(hashes.SHAKE128(digest_size=32)) + + +@pytest.mark.supported( + only_if=lambda backend: rust_openssl.CRYPTOGRAPHY_OPENSSL_330_OR_GREATER, + skip_message="Requires backend with XOF support", +) +class TestXOFHash: + def test_hash_reject_unicode(self, backend): + m = hashes.XOFHash(hashes.SHAKE128(sys.maxsize)) + with pytest.raises(TypeError): + m.update("\u00fc") # type: ignore[arg-type] + + def test_incorrect_hash_algorithm_type(self, backend): + with pytest.raises(TypeError): + # Instance required + hashes.XOFHash(hashes.SHAKE128) # type: ignore[arg-type] + + with pytest.raises(TypeError): + hashes.XOFHash(hashes.SHA256()) # type: ignore[arg-type] + + def test_raises_update_after_squeeze(self, backend): + h = hashes.XOFHash(hashes.SHAKE128(digest_size=256)) + h.update(b"foo") + h.squeeze(5) + + with pytest.raises(AlreadyFinalized): + h.update(b"bar") + + def test_copy(self, backend): + h = hashes.XOFHash(hashes.SHAKE128(digest_size=256)) + h.update(b"foo") + h.update(b"bar") + h2 = h.copy() + assert h2.squeeze(10) == h.squeeze(10) + + def test_exhaust_bytes(self, backend): + h = hashes.XOFHash(hashes.SHAKE128(digest_size=256)) + h.update(b"foo") + with pytest.raises(ValueError): + h.squeeze(257) + h.squeeze(200) + h.squeeze(56) + with pytest.raises(ValueError): + h.squeeze(1) + + +@pytest.mark.supported( + only_if=lambda backend: rust_openssl.CRYPTOGRAPHY_OPENSSL_330_OR_GREATER, + skip_message="Requires backend with XOF support", +) +class TestXOFSHAKE128: + def test_shake128_variable(self, backend, subtests): + vectors = _load_all_params( + os.path.join("hashes", "SHAKE"), + ["SHAKE128VariableOut.rsp"], + load_nist_vectors, + ) + for vector in vectors: + with subtests.test(): + output_length = int(vector["outputlen"]) // 8 + msg = binascii.unhexlify(vector["msg"]) + shake = hashes.SHAKE128(digest_size=output_length) + m = hashes.XOFHash(shake) + m.update(msg) + remaining = output_length + data = b"" + stride = random.randint(1, 128) + while remaining > 0: + stride = remaining if remaining < stride else stride + data += m.squeeze(stride) + remaining -= stride + assert data == binascii.unhexlify(vector["output"]) + + +@pytest.mark.supported( + only_if=lambda backend: rust_openssl.CRYPTOGRAPHY_OPENSSL_330_OR_GREATER, + skip_message="Requires backend with XOF support", +) +class TestXOFSHAKE256: + def test_shake256_variable(self, backend, subtests): + vectors = _load_all_params( + os.path.join("hashes", "SHAKE"), + ["SHAKE256VariableOut.rsp"], + load_nist_vectors, + ) + for vector in vectors: + with subtests.test(): + output_length = int(vector["outputlen"]) // 8 + msg = binascii.unhexlify(vector["msg"]) + shake = hashes.SHAKE256(digest_size=output_length) + m = hashes.XOFHash(shake) + m.update(msg) + remaining = output_length + data = b"" + stride = random.randint(1, 128) + while remaining > 0: + stride = remaining if remaining < stride else stride + data += m.squeeze(stride) + remaining -= stride + assert data == binascii.unhexlify(vector["output"])