Archive-Team
/
zfs
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 103 Wiki Activity

Add generic implementation handling and SHA2 impl 

The skeleton file module/icp/include/generic_impl.c can be used for
iterating over different implementations of algorithms.

It is used by SHA256, SHA512 and BLAKE3 currently.

The Solaris SHA2 implementation got replaced with a version which is
based on public domain code of cppcrypto v0.10.

These assembly files are taken from current openssl master:
- sha256-x86_64.S: x64, SSSE3, AVX, AVX2, SHA-NI (x86_64)
- sha512-x86_64.S: x64, AVX, AVX2 (x86_64)
- sha256-armv7.S: ARMv7, NEON, ARMv8-CE (arm)
- sha512-armv7.S: ARMv7, NEON (arm)
- sha256-armv8.S: ARMv7, NEON, ARMv8-CE (aarch64)
- sha512-armv8.S: ARMv7, ARMv8-CE (aarch64)
- sha256-ppc.S: Generic PPC64 LE/BE (ppc64)
- sha512-ppc.S: Generic PPC64 LE/BE (ppc64)
- sha256-p8.S: Power8 ISA Version 2.07 LE/BE (ppc64)
- sha512-p8.S: Power8 ISA Version 2.07 LE/BE (ppc64)

Tested-by: Rich Ercolani <rincebrain@gmail.com>
Tested-by: Sebastian Gottschall <s.gottschall@dd-wrt.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Tino Reichardt <milky-zfs@mcmilk.de>
Closes #13741
This commit is contained in:
Tino Reichardt 2023-03-01 09:40:28 +01:00 committed by Brian Behlendorf
parent ac678c8eee
commit 4c5fec01a4
30 changed files with 27986 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/Makefile.am
View File

@ -75,6 +75,7 @@ COMMON_H = \
sys/rrwlock.h \
sys/sa.h \
sys/sa_impl.h \
sys/sha2.h \
sys/skein.h \
sys/spa.h \
sys/spa_checkpoint.h \
@ -124,6 +125,7 @@ COMMON_H = \
sys/zfs_delay.h \
sys/zfs_file.h \
sys/zfs_fuid.h \
sys/zfs_impl.h \
sys/zfs_project.h \
sys/zfs_quota.h \
sys/zfs_racct.h \

127
include/sys/sha2.h Normal file
View File

@ -0,0 +1,127 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or https://opensource.org/licenses/CDDL-1.0.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
*/
#ifndef _SYS_SHA2_H
#define _SYS_SHA2_H
#ifdef _KERNEL
#include <sys/types.h>
#else
#include <stdint.h>
#include <stdlib.h>
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define SHA224_BLOCK_LENGTH 64
#define SHA256_BLOCK_LENGTH 64
#define SHA384_BLOCK_LENGTH 128
#define SHA512_BLOCK_LENGTH 128
#define SHA224_DIGEST_LENGTH 28
#define SHA256_DIGEST_LENGTH 32
#define SHA384_DIGEST_LENGTH 48
#define SHA512_DIGEST_LENGTH 64
#define SHA512_224_DIGEST_LENGTH 28
#define SHA512_256_DIGEST_LENGTH 32
#define SHA256_HMAC_BLOCK_SIZE 64
#define SHA512_HMAC_BLOCK_SIZE 128
/* sha256 context */
typedef struct {
uint32_t state[8];
uint64_t count[2];
uint8_t wbuf[64];
/* const sha256_ops_t *ops */
const void *ops;
} sha256_ctx;
/* sha512 context */
typedef struct {
uint64_t state[8];
uint64_t count[2];
uint8_t wbuf[128];
/* const sha256_ops_t *ops */
const void *ops;
} sha512_ctx;
/* SHA2 context */
typedef struct {
union {
sha256_ctx sha256;
sha512_ctx sha512;
};
/* algorithm type */
int algotype;
} SHA2_CTX;
/* SHA2 algorithm types */
typedef enum sha2_mech_type {
SHA256_MECH_INFO_TYPE, /* SUN_CKM_SHA256 */
SHA256_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA256_HMAC */
SHA256_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA256_HMAC_GENERAL */
SHA384_MECH_INFO_TYPE, /* SUN_CKM_SHA384 */
SHA384_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA384_HMAC */
SHA384_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA384_HMAC_GENERAL */
SHA512_MECH_INFO_TYPE, /* SUN_CKM_SHA512 */
SHA512_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA512_HMAC */
SHA512_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA512_HMAC_GENERAL */
SHA512_224_MECH_INFO_TYPE, /* SUN_CKM_SHA512_224 */
SHA512_256_MECH_INFO_TYPE /* SUN_CKM_SHA512_256 */
} sha2_mech_type_t;
#define SHA256 0
#define SHA256_HMAC 1
#define SHA256_HMAC_GEN 2
#define SHA384 3
#define SHA384_HMAC 4
#define SHA384_HMAC_GEN 5
#define SHA512 6
#define SHA512_HMAC 7
#define SHA512_HMAC_GEN 8
#define SHA512_224 9
#define SHA512_256 10
/* SHA2 Init function */
extern void SHA2Init(int algotype, SHA2_CTX *ctx);
/* SHA2 Update function */
extern void SHA2Update(SHA2_CTX *ctx, const void *data, size_t len);
/* SHA2 Final function */
extern void SHA2Final(void *digest, SHA2_CTX *ctx);
#ifdef __cplusplus
}
#endif
#endif /* SYS_SHA2_H */

69
include/sys/zfs_impl.h Normal file
View File

@ -0,0 +1,69 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or https://opensource.org/licenses/CDDL-1.0.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
*/
#ifndef _SYS_ZFS_IMPL_H
#define _SYS_ZFS_IMPL_H
#ifdef __cplusplus
extern "C" {
#endif
/* generic implementation backends */
typedef struct
{
/* algorithm name */
const char *name;
/* get number of supported implementations */
uint32_t (*getcnt)(void);
/* get id of selected implementation */
uint32_t (*getid)(void);
/* get name of selected implementation */
const char *(*getname)(void);
/* setup id as fastest implementation */
void (*set_fastest)(uint32_t id);
/* set implementation by id */
void (*setid)(uint32_t id);
/* set implementation by name */
int (*setname)(const char *val);
} zfs_impl_t;
/* return some set of function pointer */
extern const zfs_impl_t *zfs_impl_get_ops(const char *algo);
extern const zfs_impl_t zfs_blake3_ops;
extern const zfs_impl_t zfs_sha256_ops;
extern const zfs_impl_t zfs_sha512_ops;
#ifdef __cplusplus
}
#endif
#endif /* _SYS_ZFS_IMPL_H */

6
include/sys/zio_checksum.h
View File

@ -110,9 +110,9 @@ _SYS_ZIO_CHECKSUM_H zio_checksum_info_t
*/
/* SHA2 */
extern zio_checksum_t abd_checksum_SHA256;
extern zio_checksum_t abd_checksum_SHA512_native;
extern zio_checksum_t abd_checksum_SHA512_byteswap;
extern zio_checksum_t abd_checksum_sha256;
extern zio_checksum_t abd_checksum_sha512_native;
extern zio_checksum_t abd_checksum_sha512_byteswap;
/* Skein */
extern zio_checksum_t abd_checksum_skein_native;

28
lib/libicp/Makefile.am
View File

@ -16,7 +16,6 @@ nodist_libicp_la_SOURCES = \
module/icp/algs/blake3/blake3.c \
module/icp/algs/blake3/blake3_generic.c \
module/icp/algs/blake3/blake3_impl.c \
module/icp/algs/blake3/blake3_x86-64.c \
module/icp/algs/edonr/edonr.c \
module/icp/algs/modes/modes.c \
module/icp/algs/modes/cbc.c \
@ -26,7 +25,9 @@ nodist_libicp_la_SOURCES = \
module/icp/algs/modes/ctr.c \
module/icp/algs/modes/ccm.c \
module/icp/algs/modes/ecb.c \
module/icp/algs/sha2/sha2.c \
module/icp/algs/sha2/sha2_generic.c \
module/icp/algs/sha2/sha256_impl.c \
module/icp/algs/sha2/sha512_impl.c \
module/icp/algs/skein/skein.c \
module/icp/algs/skein/skein_block.c \
module/icp/algs/skein/skein_iv.c \
@ -38,18 +39,31 @@ nodist_libicp_la_SOURCES = \
module/icp/core/kcf_prov_lib.c \
module/icp/core/kcf_callprov.c \
module/icp/core/kcf_mech_tabs.c \
module/icp/core/kcf_prov_tabs.c
module/icp/core/kcf_prov_tabs.c \
module/zfs/zfs_impl.c
if TARGET_CPU_AARCH64
nodist_libicp_la_SOURCES += \
module/icp/asm-aarch64/blake3/b3_aarch64_sse2.S \
module/icp/asm-aarch64/blake3/b3_aarch64_sse41.S
module/icp/asm-aarch64/blake3/b3_aarch64_sse41.S \
module/icp/asm-aarch64/sha2/sha256-armv8.S \
module/icp/asm-aarch64/sha2/sha512-armv8.S
endif
if TARGET_CPU_ARM
nodist_libicp_la_SOURCES += \
module/icp/asm-arm/sha2/sha256-armv7.S \
module/icp/asm-arm/sha2/sha512-armv7.S
endif
if TARGET_CPU_POWERPC
nodist_libicp_la_SOURCES += \
module/icp/asm-ppc64/blake3/b3_ppc64le_sse2.S \
module/icp/asm-ppc64/blake3/b3_ppc64le_sse41.S
module/icp/asm-ppc64/blake3/b3_ppc64le_sse41.S \
module/icp/asm-ppc64/sha2/sha256-ppc.S \
module/icp/asm-ppc64/sha2/sha512-ppc.S \
module/icp/asm-ppc64/sha2/sha256-p8.S \
module/icp/asm-ppc64/sha2/sha512-p8.S
endif
if TARGET_CPU_X86_64
@ -60,8 +74,8 @@ nodist_libicp_la_SOURCES += \
module/icp/asm-x86_64/modes/gcm_pclmulqdq.S \
module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S \
module/icp/asm-x86_64/modes/ghash-x86_64.S \
module/icp/asm-x86_64/sha2/sha256_impl.S \
module/icp/asm-x86_64/sha2/sha512_impl.S \
module/icp/asm-x86_64/sha2/sha256-x86_64.S \
module/icp/asm-x86_64/sha2/sha512-x86_64.S \
module/icp/asm-x86_64/blake3/blake3_avx2.S \
module/icp/asm-x86_64/blake3/blake3_avx512.S \
module/icp/asm-x86_64/blake3/blake3_sse2.S \

3
lib/libzfs/Makefile.am
View File

@ -34,8 +34,6 @@ dist_libzfs_la_SOURCES += \
endif
nodist_libzfs_la_SOURCES = \
module/icp/algs/sha2/sha2.c \
\
module/zcommon/cityhash.c \
module/zcommon/zfeature_common.c \
module/zcommon/zfs_comutil.c \
@ -52,7 +50,6 @@ nodist_libzfs_la_SOURCES = \
module/zcommon/zpool_prop.c \
module/zcommon/zprop_common.c
libzfs_la_LIBADD = \
libshare.la \
libzfs_core.la \

2
lib/libzpool/Makefile.am
View File

@ -118,7 +118,7 @@ nodist_libzpool_la_SOURCES = \
module/zfs/refcount.c \
module/zfs/rrwlock.c \
module/zfs/sa.c \
module/zfs/sha256.c \
module/zfs/sha2_zfs.c \
module/zfs/skein_zfs.c \
module/zfs/spa.c \
module/zfs/spa_checkpoint.c \

28
module/Kbuild.in
View File

@ -85,7 +85,6 @@ ICP_OBJS := \
algs/blake3/blake3.o \
algs/blake3/blake3_generic.o \
algs/blake3/blake3_impl.o \
algs/blake3/blake3_x86-64.o \
algs/edonr/edonr.o \
algs/modes/cbc.o \
algs/modes/ccm.o \
@ -94,6 +93,9 @@ ICP_OBJS := \
algs/modes/gcm.o \
algs/modes/gcm_generic.o \
algs/modes/modes.o \
algs/sha2/sha2_generic.o \
algs/sha2/sha256_impl.o \
algs/sha2/sha512_impl.o \
algs/skein/skein.o \
algs/skein/skein_block.o \
algs/skein/skein_iv.o \
@ -119,30 +121,40 @@ ICP_OBJS_X86_64 := \
asm-x86_64/blake3/blake3_avx512.o \
asm-x86_64/blake3/blake3_sse2.o \
asm-x86_64/blake3/blake3_sse41.o \
asm-x86_64/sha2/sha256-x86_64.o \
asm-x86_64/sha2/sha512-x86_64.o \
asm-x86_64/modes/aesni-gcm-x86_64.o \
asm-x86_64/modes/gcm_pclmulqdq.o \
asm-x86_64/modes/ghash-x86_64.o
ICP_OBJS_X86 := \
algs/aes/aes_impl_aesni.o \
algs/aes/aes_impl_x86-64.o \
algs/modes/gcm_pclmulqdq.o
ICP_OBJS_ARM := \
asm-arm/sha2/sha256-armv7.o \
asm-arm/sha2/sha512-armv7.o
ICP_OBJS_ARM64 := \
asm-aarch64/blake3/b3_aarch64_sse2.o \
asm-aarch64/blake3/b3_aarch64_sse41.o
asm-aarch64/blake3/b3_aarch64_sse41.o \
asm-aarch64/sha2/sha256-armv8.o \
asm-aarch64/sha2/sha512-armv8.o
ICP_OBJS_PPC_PPC64 := \
asm-ppc64/blake3/b3_ppc64le_sse2.o \
asm-ppc64/blake3/b3_ppc64le_sse41.o
asm-ppc64/blake3/b3_ppc64le_sse41.o \
asm-ppc64/sha2/sha256-p8.o \
asm-ppc64/sha2/sha512-p8.o \
asm-ppc64/sha2/sha256-ppc.o \
asm-ppc64/sha2/sha512-ppc.o
zfs-objs += $(addprefix icp/,$(ICP_OBJS))
zfs-$(CONFIG_X86) += $(addprefix icp/,$(ICP_OBJS_X86))
zfs-$(CONFIG_UML_X86)+= $(addprefix icp/,$(ICP_OBJS_X86))
zfs-$(CONFIG_X86_64) += $(addprefix icp/,$(ICP_OBJS_X86_64))
zfs-$(CONFIG_ARM) += $(addprefix icp/,$(ICP_OBJS_ARM))
zfs-$(CONFIG_ARM64) += $(addprefix icp/,$(ICP_OBJS_ARM64))
zfs-$(CONFIG_PPC) += $(addprefix icp/,$(ICP_OBJS_PPC_PPC64))
zfs-$(CONFIG_PPC64) += $(addprefix icp/,$(ICP_OBJS_PPC_PPC64))
@ -156,6 +168,11 @@ $(addprefix $(obj)/icp/,$(ICP_OBJS) $(ICP_OBJS_X86) $(ICP_OBJS_X86_64) \
# Suppress objtool "return with modified stack frame" warnings.
OBJECT_FILES_NON_STANDARD_aesni-gcm-x86_64.o := y
# Suppress objtool "unsupported stack pointer realignment" warnings.
# See #6950 for the reasoning.
OBJECT_FILES_NON_STANDARD_sha256-x86_64.o := y
OBJECT_FILES_NON_STANDARD_sha512-x86_64.o := y
LUA_OBJS := \
lapi.o \
lauxlib.o \
@ -382,6 +399,7 @@ ZFS_OBJS := \
zfs_chksum.o \
zfs_fm.o \
zfs_fuid.o \
zfs_impl.o \
zfs_ioctl.o \
zfs_log.o \
zfs_onexit.o \

36
module/Makefile.bsd
View File

@ -13,10 +13,15 @@ KMOD= openzfs
${SRCDIR}/lua \
${SRCDIR}/nvpair \
${SRCDIR}/icp/algs/blake3 \
${SRCDIR}/icp/algs/edonr \
${SRCDIR}/icp/algs/sha2 \
${SRCDIR}/icp/asm-aarch64/blake3 \
${SRCDIR}/icp/asm-aarch64/sha2 \
${SRCDIR}/icp/asm-arm/sha2 \
${SRCDIR}/icp/asm-ppc64/sha2 \
${SRCDIR}/icp/asm-ppc64/blake3 \
${SRCDIR}/icp/asm-x86_64/blake3 \
${SRCDIR}/icp/algs/edonr \
${SRCDIR}/icp/asm-x86_64/sha2 \
${SRCDIR}/os/freebsd/spl \
${SRCDIR}/os/freebsd/zfs \
${SRCDIR}/unicode \
@ -27,8 +32,6 @@ KMOD= openzfs
${SRCDIR}/zstd/lib/compress \
${SRCDIR}/zstd/lib/decompress
CFLAGS+= -I${INCDIR}
CFLAGS+= -I${INCDIR}/os/freebsd
CFLAGS+= -I${INCDIR}/os/freebsd/spl
@ -88,8 +91,7 @@ SRCS+= edonr.c
#icp/algs/blake3
SRCS+= blake3.c \
blake3_generic.c \
blake3_impl.c \
blake3_x86-64.c
blake3_impl.c
#icp/asm-aarch64/blake3
SRCS+= b3_aarch64_sse2.S \
@ -105,6 +107,29 @@ SRCS+= blake3_avx2.S \
blake3_sse2.S \
blake3_sse41.S
#icp/algs/sha2
SRCS+= sha2_generic.c \
sha256_impl.c \
sha512_impl.c
#icp/asm-arm/sha2
SRCS+= sha256-armv7.S \
sha512-armv7.S
#icp/asm-aarch64/sha2
SRCS+= sha256-armv8.S \
sha512-armv8.S
#icp/asm-ppc64/sha2
SRCS+= sha256-p8.S \
sha512-p8.S \
sha256-ppc.S \
sha512-ppc.S
#icp/asm-x86_64/sha2
SRCS+= sha256-x86_64.S \
sha512-x86_64.S
#lua
SRCS+= lapi.c \
lauxlib.c \
@ -320,6 +345,7 @@ SRCS+= abd.c \
zfs_file_os.c \
zfs_fm.c \
zfs_fuid.c \
zfs_impl.c \
zfs_ioctl.c \
zfs_log.c \
zfs_onexit.c \

299
module/icp/algs/sha2/sha256_impl.c Normal file
View File

@ -0,0 +1,299 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or https://opensource.org/licenses/CDDL-1.0.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
*/
#include <sys/zfs_context.h>
#include <sys/zfs_impl.h>
#include <sys/sha2.h>
#include <sys/simd.h>
#include <sha2/sha2_impl.h>
#define TF(E, N) \
extern void E(uint32_t s[8], const void *, size_t); \
static inline void N(uint32_t s[8], const void *d, size_t b) { \
kfpu_begin(); E(s, d, b); kfpu_end(); \
}
/* some implementation is always okay */
static inline boolean_t sha2_is_supported(void)
{
return (B_TRUE);
}
#if defined(__x86_64)
extern void zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t);
const sha256_ops_t sha256_x64_impl = {
.is_supported = sha2_is_supported,
.transform = zfs_sha256_transform_x64,
.name = "x64"
};
#if defined(HAVE_SSSE3)
static boolean_t sha2_have_ssse3(void)
{
return (kfpu_allowed() && zfs_ssse3_available());
}
TF(zfs_sha256_transform_ssse3, tf_sha256_ssse3);
const sha256_ops_t sha256_ssse3_impl = {
.is_supported = sha2_have_ssse3,
.transform = tf_sha256_ssse3,
.name = "ssse3"
};
#endif
#if defined(HAVE_AVX)
static boolean_t sha2_have_avx(void)
{
return (kfpu_allowed() && zfs_avx_available());
}
TF(zfs_sha256_transform_avx, tf_sha256_avx);
const sha256_ops_t sha256_avx_impl = {
.is_supported = sha2_have_avx,
.transform = tf_sha256_avx,
.name = "avx"
};
#endif
#if defined(HAVE_AVX2)
static boolean_t sha2_have_avx2(void)
{
return (kfpu_allowed() && zfs_avx2_available());
}
TF(zfs_sha256_transform_avx2, tf_sha256_avx2);
const sha256_ops_t sha256_avx2_impl = {
.is_supported = sha2_have_avx2,
.transform = tf_sha256_avx2,
.name = "avx2"
};
#endif
#if defined(HAVE_SSE4_1)
static boolean_t sha2_have_shani(void)
{
return (kfpu_allowed() && zfs_sse4_1_available() && \
zfs_shani_available());
}
TF(zfs_sha256_transform_shani, tf_sha256_shani);
const sha256_ops_t sha256_shani_impl = {
.is_supported = sha2_have_shani,
.transform = tf_sha256_shani,
.name = "shani"
};
#endif
#elif defined(__aarch64__) || defined(__arm__)
static boolean_t sha256_have_neon(void)
{
return (kfpu_allowed() && zfs_neon_available());
}
static boolean_t sha256_have_armv8ce(void)
{
return (kfpu_allowed() && zfs_sha256_available());
}
extern void zfs_sha256_block_armv7(uint32_t s[8], const void *, size_t);
const sha256_ops_t sha256_armv7_impl = {
.is_supported = sha2_is_supported,
.transform = zfs_sha256_block_armv7,
.name = "armv7"
};
TF(zfs_sha256_block_neon, tf_sha256_neon);
const sha256_ops_t sha256_neon_impl = {
.is_supported = sha256_have_neon,
.transform = tf_sha256_neon,
.name = "neon"
};
TF(zfs_sha256_block_armv8, tf_sha256_armv8ce);
const sha256_ops_t sha256_armv8_impl = {
.is_supported = sha256_have_armv8ce,
.transform = tf_sha256_armv8ce,
.name = "armv8-ce"
};
#elif defined(__PPC64__)
static boolean_t sha256_have_vsx(void)
{
return (kfpu_allowed() && zfs_vsx_available());
}
TF(zfs_sha256_ppc, tf_sha256_ppc);
const sha256_ops_t sha256_ppc_impl = {
.is_supported = sha2_is_supported,
.transform = tf_sha256_ppc,
.name = "ppc"
};
TF(zfs_sha256_power8, tf_sha256_power8);
const sha256_ops_t sha256_power8_impl = {
.is_supported = sha256_have_vsx,
.transform = tf_sha256_power8,
.name = "power8"
};
#endif /* __PPC64__ */
/* the two generic ones */
extern const sha256_ops_t sha256_generic_impl;
/* array with all sha256 implementations */
static const sha256_ops_t *const sha256_impls[] = {
&sha256_generic_impl,
#if defined(__x86_64)
&sha256_x64_impl,
#endif
#if defined(__x86_64) && defined(HAVE_SSSE3)
&sha256_ssse3_impl,
#endif
#if defined(__x86_64) && defined(HAVE_AVX)
&sha256_avx_impl,
#endif
#if defined(__x86_64) && defined(HAVE_AVX2)
&sha256_avx2_impl,
#endif
#if defined(__x86_64) && defined(HAVE_SSE4_1)
&sha256_shani_impl,
#endif
#if defined(__aarch64__) || defined(__arm__)
&sha256_armv7_impl,
&sha256_neon_impl,
&sha256_armv8_impl,
#endif
#if defined(__PPC64__)
&sha256_ppc_impl,
&sha256_power8_impl,
#endif /* __PPC64__ */
};
/* use the generic implementation functions */
#define IMPL_NAME "sha256"
#define IMPL_OPS_T sha256_ops_t
#define IMPL_ARRAY sha256_impls
#define IMPL_GET_OPS sha256_get_ops
#define ZFS_IMPL_OPS zfs_sha256_ops
#include <generic_impl.c>
#ifdef _KERNEL
#define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
#if defined(__linux__)
static int
sha256_param_get(char *buffer, zfs_kernel_param_t *unused)
{
const uint32_t impl = IMPL_READ(generic_impl_chosen);
char *fmt;
int cnt = 0;
/* cycling */
fmt = IMPL_FMT(impl, IMPL_CYCLE);
cnt += sprintf(buffer + cnt, fmt, "cycle");
/* list fastest */
fmt = IMPL_FMT(impl, IMPL_FASTEST);
cnt += sprintf(buffer + cnt, fmt, "fastest");
/* list all supported implementations */
generic_impl_init();
for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
fmt = IMPL_FMT(impl, i);
cnt += sprintf(buffer + cnt, fmt,
generic_supp_impls[i]->name);
}
return (cnt);
}
static int
sha256_param_set(const char *val, zfs_kernel_param_t *unused)
{
(void) unused;
return (generic_impl_setname(val));
}
#elif defined(__FreeBSD__)
#include <sys/sbuf.h>
static int
sha256_param(ZFS_MODULE_PARAM_ARGS)
{
int err;
generic_impl_init();
if (req->newptr == NULL) {
const uint32_t impl = IMPL_READ(generic_impl_chosen);
const int init_buflen = 64;
const char *fmt;
struct sbuf *s;
s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
/* cycling */
fmt = IMPL_FMT(impl, IMPL_CYCLE);
(void) sbuf_printf(s, fmt, "cycle");
/* list fastest */
fmt = IMPL_FMT(impl, IMPL_FASTEST);
(void) sbuf_printf(s, fmt, "fastest");
/* list all supported implementations */
for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
fmt = IMPL_FMT(impl, i);
(void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
}
err = sbuf_finish(s);
sbuf_delete(s);
return (err);
}
char buf[16];
err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
if (err) {
return (err);
}
return (-generic_impl_setname(buf));
}
#endif
#undef IMPL_FMT
ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha256_impl,
sha256_param_set, sha256_param_get, ZMOD_RW, \
"Select SHA256 implementation.");
#endif
#undef TF

562
module/icp/algs/sha2/sha2_generic.c Normal file
View File

@ -0,0 +1,562 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or https://opensource.org/licenses/CDDL-1.0.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Based on public domain code in cppcrypto 0.10.
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
*/
#include <sys/zfs_context.h>
#include <sys/zfs_impl.h>
#include <sys/sha2.h>
#include <sha2/sha2_impl.h>
/*
* On i386, gcc brings this for sha512_generic():
* error: the frame size of 1040 bytes is larger than 1024
*/
#if defined(__GNUC__) && defined(_ILP32)
#pragma GCC diagnostic ignored "-Wframe-larger-than="
#endif
/* SHA256 */
static const uint32_t SHA256_K[64] = {
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
};
#define Ch(x, y, z) ((z) ^ ((x) & ((y) ^ (z))))
#define Maj(x, y, z) (((y) & (z)) | (((y) | (z)) & (x)))
#define rotr32(x, n) (((x) >> n) | ((x) << (32 - n)))
#define sum0(x) (rotr32((x), 2) ^ rotr32((x), 13) ^ rotr32((x), 22))
#define sum1(x) (rotr32((x), 6) ^ rotr32((x), 11) ^ rotr32((x), 25))
#define sigma0(x) (rotr32((x), 7) ^ rotr32((x), 18) ^ ((x) >> 3))
#define sigma1(x) (rotr32((x), 17) ^ rotr32((x), 19) ^ ((x) >> 10))
#define WU(j) (W[j & 15] += sigma1(W[(j + 14) & 15]) \
+ W[(j + 9) & 15] + sigma0(W[(j + 1) & 15]))
#define COMPRESS(i, j, K) \
T1 = h + sum1(e) + Ch(e, f, g) + K[i + j] + (i? WU(j): W[j]); \
T2 = sum0(a) + Maj(a, b, c); \
h = g, g = f, f = e, e = d + T1; \
d = c, c = b, b = a, a = T1 + T2;
static void sha256_generic(uint32_t state[8], const void *data, size_t num_blks)
{
uint64_t blk;
for (blk = 0; blk < num_blks; blk++) {
uint32_t W[16];
uint32_t a, b, c, d, e, f, g, h;
uint32_t T1, T2;
int i;
for (i = 0; i < 16; i++) {
W[i] = BE_32( \
(((const uint32_t *)(data))[blk * 16 + i]));
}
a = state[0];
b = state[1];
c = state[2];
d = state[3];
e = state[4];
f = state[5];
g = state[6];
h = state[7];
for (i = 0; i <= 63; i += 16) {
COMPRESS(i, 0, SHA256_K);
COMPRESS(i, 1, SHA256_K);
COMPRESS(i, 2, SHA256_K);
COMPRESS(i, 3, SHA256_K);
COMPRESS(i, 4, SHA256_K);
COMPRESS(i, 5, SHA256_K);
COMPRESS(i, 6, SHA256_K);
COMPRESS(i, 7, SHA256_K);
COMPRESS(i, 8, SHA256_K);
COMPRESS(i, 9, SHA256_K);
COMPRESS(i, 10, SHA256_K);
COMPRESS(i, 11, SHA256_K);
COMPRESS(i, 12, SHA256_K);
COMPRESS(i, 13, SHA256_K);
COMPRESS(i, 14, SHA256_K);
COMPRESS(i, 15, SHA256_K);
}
state[0] += a;
state[1] += b;
state[2] += c;
state[3] += d;
state[4] += e;
state[5] += f;
state[6] += g;
state[7] += h;
}
}
#undef sum0
#undef sum1
#undef sigma0
#undef sigma1
#define rotr64(x, n) (((x) >> n) | ((x) << (64 - n)))
#define sum0(x) (rotr64((x), 28) ^ rotr64((x), 34) ^ rotr64((x), 39))
#define sum1(x) (rotr64((x), 14) ^ rotr64((x), 18) ^ rotr64((x), 41))
#define sigma0(x) (rotr64((x), 1) ^ rotr64((x), 8) ^ ((x) >> 7))
#define sigma1(x) (rotr64((x), 19) ^ rotr64((x), 61) ^ ((x) >> 6))
/* SHA512 */
static const uint64_t SHA512_K[80] = {
0x428a2f98d728ae22, 0x7137449123ef65cd, 0xb5c0fbcfec4d3b2f,
0xe9b5dba58189dbbc, 0x3956c25bf348b538, 0x59f111f1b605d019,
0x923f82a4af194f9b, 0xab1c5ed5da6d8118, 0xd807aa98a3030242,
0x12835b0145706fbe, 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2,
0x72be5d74f27b896f, 0x80deb1fe3b1696b1, 0x9bdc06a725c71235,
0xc19bf174cf692694, 0xe49b69c19ef14ad2, 0xefbe4786384f25e3,
0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65, 0x2de92c6f592b0275,
0x4a7484aa6ea6e483, 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5,
0x983e5152ee66dfab, 0xa831c66d2db43210, 0xb00327c898fb213f,
0xbf597fc7beef0ee4, 0xc6e00bf33da88fc2, 0xd5a79147930aa725,
0x06ca6351e003826f, 0x142929670a0e6e70, 0x27b70a8546d22ffc,
0x2e1b21385c26c926, 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df,
0x650a73548baf63de, 0x766a0abb3c77b2a8, 0x81c2c92e47edaee6,
0x92722c851482353b, 0xa2bfe8a14cf10364, 0xa81a664bbc423001,
0xc24b8b70d0f89791, 0xc76c51a30654be30, 0xd192e819d6ef5218,
0xd69906245565a910, 0xf40e35855771202a, 0x106aa07032bbd1b8,
0x19a4c116b8d2d0c8, 0x1e376c085141ab53, 0x2748774cdf8eeb99,
0x34b0bcb5e19b48a8, 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb,
0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3, 0x748f82ee5defb2fc,
0x78a5636f43172f60, 0x84c87814a1f0ab72, 0x8cc702081a6439ec,
0x90befffa23631e28, 0xa4506cebde82bde9, 0xbef9a3f7b2c67915,
0xc67178f2e372532b, 0xca273eceea26619c, 0xd186b8c721c0c207,
0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178, 0x06f067aa72176fba,
0x0a637dc5a2c898a6, 0x113f9804bef90dae, 0x1b710b35131c471b,
0x28db77f523047d84, 0x32caab7b40c72493, 0x3c9ebe0a15c9bebc,
0x431d67c49c100d4c, 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a,
0x5fcb6fab3ad6faec, 0x6c44198c4a475817
};
static void sha512_generic(uint64_t state[8], const void *data, size_t num_blks)
{
uint64_t blk;
for (blk = 0; blk < num_blks; blk++) {
uint64_t W[16];
uint64_t a, b, c, d, e, f, g, h;
uint64_t T1, T2;
int i;
for (i = 0; i < 16; i++) {
W[i] = BE_64( \
(((const uint64_t *)(data))[blk * 16 + i]));
}
a = state[0];
b = state[1];
c = state[2];
d = state[3];
e = state[4];
f = state[5];
g = state[6];
h = state[7];
for (i = 0; i <= 79; i += 16) {
COMPRESS(i, 0, SHA512_K);
COMPRESS(i, 1, SHA512_K);
COMPRESS(i, 2, SHA512_K);
COMPRESS(i, 3, SHA512_K);
COMPRESS(i, 4, SHA512_K);
COMPRESS(i, 5, SHA512_K);
COMPRESS(i, 6, SHA512_K);
COMPRESS(i, 7, SHA512_K);
COMPRESS(i, 8, SHA512_K);
COMPRESS(i, 9, SHA512_K);
COMPRESS(i, 10, SHA512_K);
COMPRESS(i, 11, SHA512_K);
COMPRESS(i, 12, SHA512_K);
COMPRESS(i, 13, SHA512_K);
COMPRESS(i, 14, SHA512_K);
COMPRESS(i, 15, SHA512_K);
}
state[0] += a;
state[1] += b;
state[2] += c;
state[3] += d;
state[4] += e;
state[5] += f;
state[6] += g;
state[7] += h;
}
}
static void sha256_update(sha256_ctx *ctx, const uint8_t *data, size_t len)
{
uint64_t pos = ctx->count[0];
uint64_t total = ctx->count[1];
uint8_t *m = ctx->wbuf;
const sha256_ops_t *ops = ctx->ops;
if (pos && pos + len >= 64) {
memcpy(m + pos, data, 64 - pos);
ops->transform(ctx->state, m, 1);
len -= 64 - pos;
total += (64 - pos) * 8;
data += 64 - pos;
pos = 0;
}
if (len >= 64) {
uint32_t blocks = len / 64;
uint32_t bytes = blocks * 64;
ops->transform(ctx->state, data, blocks);
len -= bytes;
total += (bytes) * 8;
data += bytes;
}
memcpy(m + pos, data, len);
pos += len;
total += len * 8;
ctx->count[0] = pos;
ctx->count[1] = total;
}
static void sha512_update(sha512_ctx *ctx, const uint8_t *data, size_t len)
{
uint64_t pos = ctx->count[0];
uint64_t total = ctx->count[1];
uint8_t *m = ctx->wbuf;
const sha512_ops_t *ops = ctx->ops;
if (pos && pos + len >= 128) {
memcpy(m + pos, data, 128 - pos);
ops->transform(ctx->state, m, 1);
len -= 128 - pos;
total += (128 - pos) * 8;
data += 128 - pos;
pos = 0;
}
if (len >= 128) {
uint64_t blocks = len / 128;
uint64_t bytes = blocks * 128;
ops->transform(ctx->state, data, blocks);
len -= bytes;
total += (bytes) * 8;
data += bytes;
}
memcpy(m + pos, data, len);
pos += len;
total += len * 8;
ctx->count[0] = pos;
ctx->count[1] = total;
}
static void sha256_final(sha256_ctx *ctx, uint8_t *result, int bits)
{
uint64_t mlen, pos = ctx->count[0];
uint8_t *m = ctx->wbuf;
uint32_t *R = (uint32_t *)result;
const sha256_ops_t *ops = ctx->ops;
m[pos++] = 0x80;
if (pos > 56) {
memset(m + pos, 0, 64 - pos);
ops->transform(ctx->state, m, 1);
pos = 0;
}
memset(m + pos, 0, 64 - pos);
mlen = BE_64(ctx->count[1]);
memcpy(m + (64 - 8), &mlen, 64 / 8);
ops->transform(ctx->state, m, 1);
switch (bits) {
case 224: /* 28 - unused currently /TR */
R[0] = BE_32(ctx->state[0]);
R[1] = BE_32(ctx->state[1]);
R[2] = BE_32(ctx->state[2]);
R[3] = BE_32(ctx->state[3]);
R[4] = BE_32(ctx->state[4]);
R[5] = BE_32(ctx->state[5]);
R[6] = BE_32(ctx->state[6]);
break;
case 256: /* 32 */
R[0] = BE_32(ctx->state[0]);
R[1] = BE_32(ctx->state[1]);
R[2] = BE_32(ctx->state[2]);
R[3] = BE_32(ctx->state[3]);
R[4] = BE_32(ctx->state[4]);
R[5] = BE_32(ctx->state[5]);
R[6] = BE_32(ctx->state[6]);
R[7] = BE_32(ctx->state[7]);
break;
}
memset(ctx, 0, sizeof (*ctx));
}
static void sha512_final(sha512_ctx *ctx, uint8_t *result, int bits)
{
uint64_t mlen, pos = ctx->count[0];
uint8_t *m = ctx->wbuf, *r;
uint64_t *R = (uint64_t *)result;
const sha512_ops_t *ops = ctx->ops;
m[pos++] = 0x80;
if (pos > 112) {
memset(m + pos, 0, 128 - pos);
ops->transform(ctx->state, m, 1);
pos = 0;
}
memset(m + pos, 0, 128 - pos);
mlen = BE_64(ctx->count[1]);
memcpy(m + (128 - 8), &mlen, 64 / 8);
ops->transform(ctx->state, m, 1);
switch (bits) {
case 224: /* 28 => 3,5 x 8 */
r = result + 24;
R[0] = BE_64(ctx->state[0]);
R[1] = BE_64(ctx->state[1]);
R[2] = BE_64(ctx->state[2]);
/* last 4 bytes are special here */
*r++ = (uint8_t)(ctx->state[3] >> 56);
*r++ = (uint8_t)(ctx->state[3] >> 48);
*r++ = (uint8_t)(ctx->state[3] >> 40);
*r++ = (uint8_t)(ctx->state[3] >> 32);
break;
case 256: /* 32 */
R[0] = BE_64(ctx->state[0]);
R[1] = BE_64(ctx->state[1]);
R[2] = BE_64(ctx->state[2]);
R[3] = BE_64(ctx->state[3]);
break;
case 384: /* 48 */
R[0] = BE_64(ctx->state[0]);
R[1] = BE_64(ctx->state[1]);
R[2] = BE_64(ctx->state[2]);
R[3] = BE_64(ctx->state[3]);
R[4] = BE_64(ctx->state[4]);
R[5] = BE_64(ctx->state[5]);
break;
case 512: /* 64 */
R[0] = BE_64(ctx->state[0]);
R[1] = BE_64(ctx->state[1]);
R[2] = BE_64(ctx->state[2]);
R[3] = BE_64(ctx->state[3]);
R[4] = BE_64(ctx->state[4]);
R[5] = BE_64(ctx->state[5]);
R[6] = BE_64(ctx->state[6]);
R[7] = BE_64(ctx->state[7]);
break;
}
memset(ctx, 0, sizeof (*ctx));
}
/* SHA2 Init function */
void
SHA2Init(int algotype, SHA2_CTX *ctx)
{
sha256_ctx *ctx256 = &ctx->sha256;
sha512_ctx *ctx512 = &ctx->sha512;
ASSERT3U(algotype, >=, SHA256_MECH_INFO_TYPE);
ASSERT3U(algotype, <=, SHA512_256_MECH_INFO_TYPE);
memset(ctx, 0, sizeof (*ctx));
ctx->algotype = algotype;
switch (ctx->algotype) {
case SHA256_MECH_INFO_TYPE:
case SHA256_HMAC_MECH_INFO_TYPE:
case SHA256_HMAC_GEN_MECH_INFO_TYPE:
ctx256->state[0] = 0x6a09e667;
ctx256->state[1] = 0xbb67ae85;
ctx256->state[2] = 0x3c6ef372;
ctx256->state[3] = 0xa54ff53a;
ctx256->state[4] = 0x510e527f;
ctx256->state[5] = 0x9b05688c;
ctx256->state[6] = 0x1f83d9ab;
ctx256->state[7] = 0x5be0cd19;
ctx256->count[0] = 0;
ctx256->ops = sha256_get_ops();
break;
case SHA384_MECH_INFO_TYPE:
case SHA384_HMAC_MECH_INFO_TYPE:
case SHA384_HMAC_GEN_MECH_INFO_TYPE:
ctx512->state[0] = 0xcbbb9d5dc1059ed8ULL;
ctx512->state[1] = 0x629a292a367cd507ULL;
ctx512->state[2] = 0x9159015a3070dd17ULL;
ctx512->state[3] = 0x152fecd8f70e5939ULL;
ctx512->state[4] = 0x67332667ffc00b31ULL;
ctx512->state[5] = 0x8eb44a8768581511ULL;
ctx512->state[6] = 0xdb0c2e0d64f98fa7ULL;
ctx512->state[7] = 0x47b5481dbefa4fa4ULL;
ctx512->count[0] = 0;
ctx512->count[1] = 0;
ctx512->ops = sha512_get_ops();
break;
case SHA512_MECH_INFO_TYPE:
case SHA512_HMAC_MECH_INFO_TYPE:
case SHA512_HMAC_GEN_MECH_INFO_TYPE:
ctx512->state[0] = 0x6a09e667f3bcc908ULL;
ctx512->state[1] = 0xbb67ae8584caa73bULL;
ctx512->state[2] = 0x3c6ef372fe94f82bULL;
ctx512->state[3] = 0xa54ff53a5f1d36f1ULL;
ctx512->state[4] = 0x510e527fade682d1ULL;
ctx512->state[5] = 0x9b05688c2b3e6c1fULL;
ctx512->state[6] = 0x1f83d9abfb41bd6bULL;
ctx512->state[7] = 0x5be0cd19137e2179ULL;
ctx512->count[0] = 0;
ctx512->count[1] = 0;
ctx512->ops = sha512_get_ops();
break;
case SHA512_224_MECH_INFO_TYPE:
ctx512->state[0] = 0x8c3d37c819544da2ULL;
ctx512->state[1] = 0x73e1996689dcd4d6ULL;
ctx512->state[2] = 0x1dfab7ae32ff9c82ULL;
ctx512->state[3] = 0x679dd514582f9fcfULL;
ctx512->state[4] = 0x0f6d2b697bd44da8ULL;
ctx512->state[5] = 0x77e36f7304c48942ULL;
ctx512->state[6] = 0x3f9d85a86a1d36c8ULL;
ctx512->state[7] = 0x1112e6ad91d692a1ULL;
ctx512->count[0] = 0;
ctx512->count[1] = 0;
ctx512->ops = sha512_get_ops();
break;
case SHA512_256_MECH_INFO_TYPE:
ctx512->state[0] = 0x22312194fc2bf72cULL;
ctx512->state[1] = 0x9f555fa3c84c64c2ULL;
ctx512->state[2] = 0x2393b86b6f53b151ULL;
ctx512->state[3] = 0x963877195940eabdULL;
ctx512->state[4] = 0x96283ee2a88effe3ULL;
ctx512->state[5] = 0xbe5e1e2553863992ULL;
ctx512->state[6] = 0x2b0199fc2c85b8aaULL;
ctx512->state[7] = 0x0eb72ddc81c52ca2ULL;
ctx512->count[0] = 0;
ctx512->count[1] = 0;
ctx512->ops = sha512_get_ops();
break;
}
}
/* SHA2 Update function */
void
SHA2Update(SHA2_CTX *ctx, const void *data, size_t len)
{
/* check for zero input length */
if (len == 0)
return;
ASSERT3P(data, !=, NULL);
switch (ctx->algotype) {
case SHA256_MECH_INFO_TYPE:
case SHA256_HMAC_MECH_INFO_TYPE:
case SHA256_HMAC_GEN_MECH_INFO_TYPE:
sha256_update(&ctx->sha256, data, len);
break;
case SHA384_MECH_INFO_TYPE:
case SHA384_HMAC_MECH_INFO_TYPE:
case SHA384_HMAC_GEN_MECH_INFO_TYPE:
sha512_update(&ctx->sha512, data, len);
break;
case SHA512_MECH_INFO_TYPE:
case SHA512_HMAC_MECH_INFO_TYPE:
case SHA512_HMAC_GEN_MECH_INFO_TYPE:
sha512_update(&ctx->sha512, data, len);
break;
case SHA512_224_MECH_INFO_TYPE:
sha512_update(&ctx->sha512, data, len);
break;
case SHA512_256_MECH_INFO_TYPE:
sha512_update(&ctx->sha512, data, len);
break;
}
}
/* SHA2Final function */
void
SHA2Final(void *digest, SHA2_CTX *ctx)
{
switch (ctx->algotype) {
case SHA256_MECH_INFO_TYPE:
case SHA256_HMAC_MECH_INFO_TYPE:
case SHA256_HMAC_GEN_MECH_INFO_TYPE:
sha256_final(&ctx->sha256, digest, 256);
break;
case SHA384_MECH_INFO_TYPE:
case SHA384_HMAC_MECH_INFO_TYPE:
case SHA384_HMAC_GEN_MECH_INFO_TYPE:
sha512_final(&ctx->sha512, digest, 384);
break;
case SHA512_MECH_INFO_TYPE:
case SHA512_HMAC_MECH_INFO_TYPE:
case SHA512_HMAC_GEN_MECH_INFO_TYPE:
sha512_final(&ctx->sha512, digest, 512);
break;
case SHA512_224_MECH_INFO_TYPE:
sha512_final(&ctx->sha512, digest, 224);
break;
case SHA512_256_MECH_INFO_TYPE:
sha512_final(&ctx->sha512, digest, 256);
break;
}
}
/* the generic implementation is always okay */
static boolean_t sha2_is_supported(void)
{
return (B_TRUE);
}
const sha256_ops_t sha256_generic_impl = {
.name = "generic",
.transform = sha256_generic,
.is_supported = sha2_is_supported
};
const sha512_ops_t sha512_generic_impl = {
.name = "generic",
.transform = sha512_generic,
.is_supported = sha2_is_supported
};

276
module/icp/algs/sha2/sha512_impl.c Normal file
View File

@ -0,0 +1,276 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or https://opensource.org/licenses/CDDL-1.0.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
*/
#include <sys/zfs_context.h>
#include <sys/zfs_impl.h>
#include <sys/sha2.h>
#include <sys/simd.h>
#include <sha2/sha2_impl.h>
#define TF(E, N) \
extern void E(uint64_t s[8], const void *, size_t); \
static inline void N(uint64_t s[8], const void *d, size_t b) { \
kfpu_begin(); E(s, d, b); kfpu_end(); \
}
/* some implementation is always okay */
static inline boolean_t sha2_is_supported(void)
{
return (B_TRUE);
}
#if defined(__x86_64)
extern void zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t);
const sha512_ops_t sha512_x64_impl = {
.is_supported = sha2_is_supported,
.transform = zfs_sha512_transform_x64,
.name = "x64"
};
#if defined(HAVE_AVX)
static boolean_t sha2_have_avx(void)
{
return (kfpu_allowed() && zfs_avx_available());
}
TF(zfs_sha512_transform_avx, tf_sha512_avx);
const sha512_ops_t sha512_avx_impl = {
.is_supported = sha2_have_avx,
.transform = tf_sha512_avx,
.name = "avx"
};
#endif
#if defined(HAVE_AVX2)
static boolean_t sha2_have_avx2(void)
{
return (kfpu_allowed() && zfs_avx2_available());
}
TF(zfs_sha512_transform_avx2, tf_sha512_avx2);
const sha512_ops_t sha512_avx2_impl = {
.is_supported = sha2_have_avx2,
.transform = tf_sha512_avx2,
.name = "avx2"
};
#endif
#elif defined(__aarch64__)
extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t);
const sha512_ops_t sha512_armv7_impl = {
.is_supported = sha2_is_supported,
.transform = zfs_sha512_block_armv7,
.name = "armv7"
};
static boolean_t sha512_have_armv8ce(void)
{
return (kfpu_allowed() && zfs_sha512_available());
}
TF(zfs_sha512_block_armv8, tf_sha512_armv8ce);
const sha512_ops_t sha512_armv8_impl = {
.is_supported = sha512_have_armv8ce,
.transform = tf_sha512_armv8ce,
.name = "armv8-ce"
};
#elif defined(__arm__)
extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t);
const sha512_ops_t sha512_armv7_impl = {
.is_supported = sha2_is_supported,
.transform = zfs_sha512_block_armv7,
.name = "armv7"
};
static boolean_t sha512_have_neon(void)
{
return (kfpu_allowed() && zfs_neon_available());
}
TF(zfs_sha512_block_neon, tf_sha512_neon);
const sha512_ops_t sha512_neon_impl = {
.is_supported = sha512_have_neon,
.transform = tf_sha512_neon,
.name = "neon"
};
#elif defined(__PPC64__)
TF(zfs_sha512_ppc, tf_sha512_ppc);
const sha512_ops_t sha512_ppc_impl = {
.is_supported = sha2_is_supported,
.transform = tf_sha512_ppc,
.name = "ppc"
};
static boolean_t sha512_have_vsx(void)
{
return (kfpu_allowed() && zfs_vsx_available());
}
TF(zfs_sha512_power8, tf_sha512_power8);
const sha512_ops_t sha512_power8_impl = {
.is_supported = sha512_have_vsx,
.transform = tf_sha512_power8,
.name = "power8"
};
#endif /* __PPC64__ */
/* the two generic ones */
extern const sha512_ops_t sha512_generic_impl;
/* array with all sha512 implementations */
static const sha512_ops_t *const sha512_impls[] = {
&sha512_generic_impl,
#if defined(__x86_64)
&sha512_x64_impl,
#endif
#if defined(__x86_64) && defined(HAVE_AVX)
&sha512_avx_impl,
#endif
#if defined(__x86_64) && defined(HAVE_AVX2)
&sha512_avx2_impl,
#endif
#if defined(__aarch64__)
&sha512_armv7_impl,
&sha512_armv8_impl,
#endif
#if defined(__arm__)
&sha512_armv7_impl,
&sha512_neon_impl,
#endif
#if defined(__PPC64__)
&sha512_ppc_impl,
&sha512_power8_impl,
#endif /* __PPC64__ */
};
/* use the generic implementation functions */
#define IMPL_NAME "sha512"
#define IMPL_OPS_T sha512_ops_t
#define IMPL_ARRAY sha512_impls
#define IMPL_GET_OPS sha512_get_ops
#define ZFS_IMPL_OPS zfs_sha512_ops
#include <generic_impl.c>
#ifdef _KERNEL
#define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
#if defined(__linux__)
static int
sha512_param_get(char *buffer, zfs_kernel_param_t *unused)
{
const uint32_t impl = IMPL_READ(generic_impl_chosen);
char *fmt;
int cnt = 0;
/* cycling */
fmt = IMPL_FMT(impl, IMPL_CYCLE);
cnt += sprintf(buffer + cnt, fmt, "cycle");
/* list fastest */
fmt = IMPL_FMT(impl, IMPL_FASTEST);
cnt += sprintf(buffer + cnt, fmt, "fastest");
/* list all supported implementations */
generic_impl_init();
for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
fmt = IMPL_FMT(impl, i);
cnt += sprintf(buffer + cnt, fmt,
generic_supp_impls[i]->name);
}
return (cnt);
}
static int
sha512_param_set(const char *val, zfs_kernel_param_t *unused)
{
(void) unused;
return (generic_impl_setname(val));
}
#elif defined(__FreeBSD__)
#include <sys/sbuf.h>
static int
sha512_param(ZFS_MODULE_PARAM_ARGS)
{
int err;
generic_impl_init();
if (req->newptr == NULL) {
const uint32_t impl = IMPL_READ(generic_impl_chosen);
const int init_buflen = 64;
const char *fmt;
struct sbuf *s;
s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
/* cycling */
fmt = IMPL_FMT(impl, IMPL_CYCLE);
(void) sbuf_printf(s, fmt, "cycle");
/* list fastest */
fmt = IMPL_FMT(impl, IMPL_FASTEST);
(void) sbuf_printf(s, fmt, "fastest");
/* list all supported implementations */
for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
fmt = IMPL_FMT(impl, i);
(void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
}
err = sbuf_finish(s);
sbuf_delete(s);
return (err);
}
/* we got module parameter */
char buf[16];
err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
if (err) {
return (err);
}
return (-generic_impl_setname(buf));
}
#endif
#undef IMPL_FMT
ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha512_impl,
sha512_param_set, sha512_param_get, ZMOD_RW, \
"Select SHA512 implementation.");
#endif
#undef TF

1999
module/icp/asm-aarch64/sha2/sha256-armv8.S Normal file
View File

File diff suppressed because it is too large Load Diff

1558
module/icp/asm-aarch64/sha2/sha512-armv8.S Normal file
View File

File diff suppressed because it is too large Load Diff

2769
module/icp/asm-arm/sha2/sha256-armv7.S Normal file
View File

File diff suppressed because it is too large Load Diff

1822
module/icp/asm-arm/sha2/sha512-armv7.S Normal file
View File

File diff suppressed because it is too large Load Diff

1505
module/icp/asm-ppc64/sha2/sha256-p8.S Normal file
View File

File diff suppressed because it is too large Load Diff

2712
module/icp/asm-ppc64/sha2/sha256-ppc.S Normal file
View File

File diff suppressed because it is too large Load Diff

1706
module/icp/asm-ppc64/sha2/sha512-p8.S Normal file
View File

File diff suppressed because it is too large Load Diff

2958
module/icp/asm-ppc64/sha2/sha512-ppc.S Normal file
View File

File diff suppressed because it is too large Load Diff

5104
module/icp/asm-x86_64/sha2/sha256-x86_64.S Normal file
View File

File diff suppressed because it is too large Load Diff

4011
module/icp/asm-x86_64/sha2/sha512-x86_64.S Normal file
View File

File diff suppressed because it is too large Load Diff

233
module/icp/include/generic_impl.c Normal file
View File

@ -0,0 +1,233 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or https://opensource.org/licenses/CDDL-1.0.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2003, 2010 Oracle and/or its affiliates.
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
*/
/*
* This file gets included by c files for implementing the full set
* of zfs_impl.h defines.
*
* It's ment for easier maintaining multiple implementations of
* algorithms. Look into blake3_impl.c, sha256_impl.c or sha512_impl.c
* for reference.
*/
#include <sys/zfs_context.h>
#include <sys/zio_checksum.h>
#include <sys/zfs_impl.h>
/* Two default implementations */
#define IMPL_FASTEST (UINT32_MAX)
#define IMPL_CYCLE (UINT32_MAX - 1)
#define IMPL_READ(i) (*(volatile uint32_t *) &(i))
/* Implementation that contains the fastest method */
static IMPL_OPS_T generic_fastest_impl = {
.name = "fastest"
};
/* Hold all supported implementations */
static const IMPL_OPS_T *generic_supp_impls[ARRAY_SIZE(IMPL_ARRAY)];
static uint32_t generic_supp_impls_cnt = 0;
/* Currently selected implementation */
static uint32_t generic_impl_chosen = IMPL_FASTEST;
static struct generic_impl_selector {
const char *name;
uint32_t sel;
} generic_impl_selectors[] = {
{ "cycle", IMPL_CYCLE },
{ "fastest", IMPL_FASTEST }
};
/* check the supported implementations */
static void
generic_impl_init(void)
{
int i, c;
/* init only once */
if (likely(generic_supp_impls_cnt != 0))
return;
/* Move supported implementations into generic_supp_impls */
for (i = 0, c = 0; i < ARRAY_SIZE(IMPL_ARRAY); i++) {
const IMPL_OPS_T *impl = IMPL_ARRAY[i];
if (impl->is_supported && impl->is_supported())
generic_supp_impls[c++] = impl;
}
generic_supp_impls_cnt = c;
/* first init generic impl, may be changed via set_fastest() */
memcpy(&generic_fastest_impl, generic_supp_impls[0],
sizeof (generic_fastest_impl));
}
/* get number of supported implementations */
static uint32_t
generic_impl_getcnt(void)
{
generic_impl_init();
return (generic_supp_impls_cnt);
}
/* get id of selected implementation */
static uint32_t
generic_impl_getid(void)
{
generic_impl_init();
return (IMPL_READ(generic_impl_chosen));
}
/* get name of selected implementation */
static const char *
generic_impl_getname(void)
{
uint32_t impl = IMPL_READ(generic_impl_chosen);
generic_impl_init();
switch (impl) {
case IMPL_FASTEST:
return ("fastest");
case IMPL_CYCLE:
return ("cycle");
default:
return (generic_supp_impls[impl]->name);
}
}
/* set implementation by id */
static void
generic_impl_setid(uint32_t id)
{
generic_impl_init();
switch (id) {
case IMPL_FASTEST:
atomic_swap_32(&generic_impl_chosen, IMPL_FASTEST);
break;
case IMPL_CYCLE:
atomic_swap_32(&generic_impl_chosen, IMPL_CYCLE);
break;
default:
ASSERT3U(id, <, generic_supp_impls_cnt);
atomic_swap_32(&generic_impl_chosen, id);
break;
}
}
/* set implementation by name */
static int
generic_impl_setname(const char *val)
{
uint32_t impl = IMPL_READ(generic_impl_chosen);
size_t val_len;
int i, err = -EINVAL;
generic_impl_init();
val_len = strlen(val);
while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */
val_len--;
/* check mandatory implementations */
for (i = 0; i < ARRAY_SIZE(generic_impl_selectors); i++) {
const char *name = generic_impl_selectors[i].name;
if (val_len == strlen(name) &&
strncmp(val, name, val_len) == 0) {
impl = generic_impl_selectors[i].sel;
err = 0;
break;
}
}
/* check all supported implementations */
if (err != 0) {
for (i = 0; i < generic_supp_impls_cnt; i++) {
const char *name = generic_supp_impls[i]->name;
if (val_len == strlen(name) &&
strncmp(val, name, val_len) == 0) {
impl = i;
err = 0;
break;
}
}
}
if (err == 0) {
atomic_swap_32(&generic_impl_chosen, impl);
}
return (err);
}
/* setup id as fastest implementation */
static void
generic_impl_set_fastest(uint32_t id)
{
generic_impl_init();
memcpy(&generic_fastest_impl, generic_supp_impls[id],
sizeof (generic_fastest_impl));
}
/* return impl iterating functions */
const zfs_impl_t ZFS_IMPL_OPS = {
.name = IMPL_NAME,
.getcnt = generic_impl_getcnt,
.getid = generic_impl_getid,
.getname = generic_impl_getname,
.set_fastest = generic_impl_set_fastest,
.setid = generic_impl_setid,
.setname = generic_impl_setname
};
/* get impl ops_t of selected implementation */
const IMPL_OPS_T *
IMPL_GET_OPS(void)
{
const IMPL_OPS_T *ops = NULL;
uint32_t idx, impl = IMPL_READ(generic_impl_chosen);
static uint32_t cycle_count = 0;
generic_impl_init();
switch (impl) {
case IMPL_FASTEST:
ops = &generic_fastest_impl;
break;
case IMPL_CYCLE:
idx = (++cycle_count) % generic_supp_impls_cnt;
ops = generic_supp_impls[idx];
break;
default:
ASSERT3U(impl, <, generic_supp_impls_cnt);
ops = generic_supp_impls[impl];
break;
}
ASSERT3P(ops, !=, NULL);
return (ops);
}

27
module/icp/include/sha2/sha2_impl.h
View File

@ -18,9 +18,10 @@
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
* Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
*/
#ifndef _SHA2_IMPL_H
@ -32,6 +33,28 @@
extern "C" {
#endif
/* transform function definition */
typedef void (*sha256_f)(uint32_t state[8], const void *data, size_t blks);
typedef void (*sha512_f)(uint64_t state[8], const void *data, size_t blks);
/* needed for checking valid implementations */
typedef boolean_t (*sha2_is_supported_f)(void);
typedef struct {
const char *name;
sha256_f transform;
sha2_is_supported_f is_supported;
} sha256_ops_t;
typedef struct {
const char *name;
sha512_f transform;
sha2_is_supported_f is_supported;
} sha512_ops_t;
extern const sha256_ops_t *sha256_get_ops(void);
extern const sha512_ops_t *sha512_get_ops(void);
typedef enum {
SHA1_TYPE,
SHA256_TYPE,

1
module/icp/io/sha2_mod.c
View File

@ -28,7 +28,6 @@
#include <sys/crypto/common.h>
#include <sys/crypto/spi.h>
#include <sys/crypto/icp.h>
#define _SHA2_IMPL
#include <sys/sha2.h>
#include <sha2/sha2_impl.h>

14
module/zfs/sha256.c → module/zfs/sha2_zfs.c
View File

@ -18,16 +18,14 @@
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* Copyright 2013 Saso Kiselkov. All rights reserved.
* Copyright (c) 2016 by Delphix. All rights reserved.
*/
#include <sys/zfs_context.h>
#include <sys/zio.h>
#include <sys/zio_checksum.h>
#include <sys/sha2.h>
#include <sys/abd.h>
@ -42,7 +40,7 @@ sha_incremental(void *buf, size_t size, void *arg)
}
void
abd_checksum_SHA256(abd_t *abd, uint64_t size,
abd_checksum_sha256(abd_t *abd, uint64_t size,
const void *ctx_template, zio_cksum_t *zcp)
{
(void) ctx_template;
@ -79,7 +77,7 @@ bswap:
}
void
abd_checksum_SHA512_native(abd_t *abd, uint64_t size,
abd_checksum_sha512_native(abd_t *abd, uint64_t size,
const void *ctx_template, zio_cksum_t *zcp)
{
(void) ctx_template;
@ -91,12 +89,12 @@ abd_checksum_SHA512_native(abd_t *abd, uint64_t size,
}
void
abd_checksum_SHA512_byteswap(abd_t *abd, uint64_t size,
abd_checksum_sha512_byteswap(abd_t *abd, uint64_t size,
const void *ctx_template, zio_cksum_t *zcp)
{
zio_cksum_t tmp;
abd_checksum_SHA512_native(abd, size, ctx_template, &tmp);
abd_checksum_sha512_native(abd, size, ctx_template, &tmp);
zcp->zc_word[0] = BSWAP_64(tmp.zc_word[0]);
zcp->zc_word[1] = BSWAP_64(tmp.zc_word[1]);
zcp->zc_word[2] = BSWAP_64(tmp.zc_word[2]);

106
module/zfs/zfs_chksum.c
View File

@ -23,13 +23,13 @@
* Copyright (c) 2021-2022 Tino Reichardt <milky-zfs@mcmilk.de>
*/
#include <sys/types.h>
#include <sys/spa.h>
#include <sys/zio_checksum.h>
#include <sys/zfs_context.h>
#include <sys/zfs_chksum.h>
#include <sys/zfs_impl.h>
#include <sys/blake3.h>
#include <sys/sha2.h>
/* limit benchmarking to max 256KiB, when EdonR is slower then this: */
#define LIMIT_PERF_MBS 300
@ -56,25 +56,26 @@ static int chksum_stat_cnt = 0;
static kstat_t *chksum_kstat = NULL;
/*
* i3-1005G1 test output:
* Sample output on i3-1005G1 System:
*
* implementation 1k 4k 16k 64k 256k 1m 4m
* fletcher-4 5421 15001 26468 32555 34720 32801 18847
* edonr-generic 1196 1602 1761 1749 1762 1759 1751
* skein-generic 546 591 608 615 619 612 616
* sha256-generic 246 270 274 274 277 275 276
* sha256-avx 262 296 304 307 307 307 306
* sha256-sha-ni 769 1072 1172 1220 1219 1232 1228
* sha256-openssl 240 300 316 314 304 285 276
* sha512-generic 333 374 385 392 391 393 392
* sha512-openssl 353 441 467 476 472 467 426
* sha512-avx 362 444 473 475 479 476 478
* sha512-avx2 394 500 530 538 543 545 542
* blake3-generic 308 313 313 313 312 313 312
* blake3-sse2 402 1289 1423 1446 1432 1458 1413
* blake3-sse41 427 1470 1625 1704 1679 1607 1629
* blake3-avx2 428 1920 3095 3343 3356 3318 3204
* blake3-avx512 473 2687 4905 5836 5844 5643 5374
* implementation 1k 4k 16k 64k 256k 1m 4m 16m
* edonr-generic 1278 1625 1769 1776 1783 1778 1771 1767
* skein-generic 548 594 613 623 621 623 621 486
* sha256-generic 255 270 281 278 279 281 283 283
* sha256-x64 288 310 316 317 318 317 317 316
* sha256-ssse3 304 342 351 355 356 357 356 356
* sha256-avx 311 348 359 362 362 363 363 362
* sha256-avx2 330 378 389 395 395 395 395 395
* sha256-shani 908 1127 1212 1230 1233 1234 1223 1230
* sha512-generic 359 409 431 427 429 430 428 423
* sha512-x64 420 473 490 496 497 497 496 495
* sha512-avx 406 522 546 560 560 560 556 560
* sha512-avx2 464 568 601 606 609 610 607 608
* blake3-generic 330 327 324 323 324 320 323 322
* blake3-sse2 424 1366 1449 1468 1458 1453 1395 1408
* blake3-sse41 453 1554 1658 1703 1689 1669 1622 1630
* blake3-avx2 452 2013 3225 3351 3356 3261 3076 3101
* blake3-avx512 498 2869 5269 5926 5872 5643 5014 5005
*/
static int
chksum_kstat_headers(char *buf, size_t size)
@ -237,25 +238,30 @@ abort:
static void
chksum_benchmark(void)
{
#ifndef _KERNEL
/* we need the benchmark only for the kernel module */
return;
#endif
chksum_stat_t *cs;
int cbid = 0;
uint64_t max = 0;
uint32_t id, id_save;
uint64_t max;
uint32_t id, cbid = 0, id_save;
const zfs_impl_t *blake3 = zfs_impl_get_ops("blake3");
const zfs_impl_t *sha256 = zfs_impl_get_ops("sha256");
const zfs_impl_t *sha512 = zfs_impl_get_ops("sha512");
/* space for the benchmark times */
chksum_stat_cnt = 4;
chksum_stat_cnt += blake3_impl_getcnt();
/* count implementations */
chksum_stat_cnt = 2;
chksum_stat_cnt += sha256->getcnt();
chksum_stat_cnt += sha512->getcnt();
chksum_stat_cnt += blake3->getcnt();
chksum_stat_data = kmem_zalloc(
sizeof (chksum_stat_t) * chksum_stat_cnt, KM_SLEEP);
/* edonr - needs to be the first one here (slow CPU check) */
cs = &chksum_stat_data[cbid++];
/* edonr */
cs->init = abd_checksum_edonr_tmpl_init;
cs->func = abd_checksum_edonr_native;
cs->free = abd_checksum_edonr_tmpl_free;
@ -273,42 +279,58 @@ chksum_benchmark(void)
chksum_benchit(cs);
/* sha256 */
id_save = sha256->getid();
for (max = 0, id = 0; id < sha256->getcnt(); id++) {
sha256->setid(id);
cs = &chksum_stat_data[cbid++];
cs->init = 0;
cs->func = abd_checksum_SHA256;
cs->func = abd_checksum_sha256;
cs->free = 0;
cs->name = "sha256";
cs->impl = "generic";
cs->name = sha256->name;
cs->impl = sha256->getname();
chksum_benchit(cs);
if (cs->bs256k > max) {
max = cs->bs256k;
sha256->set_fastest(id);
}
}
sha256->setid(id_save);
/* sha512 */
id_save = sha512->getid();
for (max = 0, id = 0; id < sha512->getcnt(); id++) {
sha512->setid(id);
cs = &chksum_stat_data[cbid++];
cs->init = 0;
cs->func = abd_checksum_SHA512_native;
cs->func = abd_checksum_sha512_native;
cs->free = 0;
cs->name = "sha512";
cs->impl = "generic";
cs->name = sha512->name;
cs->impl = sha512->getname();
chksum_benchit(cs);
if (cs->bs256k > max) {
max = cs->bs256k;
sha512->set_fastest(id);
}
}
sha512->setid(id_save);
/* blake3 */
id_save = blake3_impl_getid();
for (id = 0; id < blake3_impl_getcnt(); id++) {
blake3_impl_setid(id);
id_save = blake3->getid();
for (max = 0, id = 0; id < blake3->getcnt(); id++) {
blake3->setid(id);
cs = &chksum_stat_data[cbid++];
cs->init = abd_checksum_blake3_tmpl_init;
cs->func = abd_checksum_blake3_native;
cs->free = abd_checksum_blake3_tmpl_free;
cs->name = "blake3";
cs->impl = blake3_impl_getname();
cs->name = blake3->name;
cs->impl = blake3->getname();
chksum_benchit(cs);
if (cs->bs256k > max) {
max = cs->bs256k;
blake3_impl_set_fastest(id);
blake3->set_fastest(id);
}
}
/* restore initial value */
blake3_impl_setid(id_save);
blake3->setid(id_save);
}
void

61
module/zfs/zfs_impl.c Normal file
View File

@ -0,0 +1,61 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or https://opensource.org/licenses/CDDL-1.0.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
*/
#include <sys/zio_checksum.h>
#include <sys/zfs_context.h>
#include <sys/zfs_impl.h>
#include <sys/blake3.h>
#include <sys/sha2.h>
/*
* impl_ops - backend for implementations of algorithms
*/
const zfs_impl_t *impl_ops[] = {
&zfs_blake3_ops,
&zfs_sha256_ops,
&zfs_sha512_ops,
NULL
};
/*
* zfs_impl_get_ops - Get the API functions for an impl backend
*/
const zfs_impl_t *
zfs_impl_get_ops(const char *algo)
{
const zfs_impl_t **ops = impl_ops;
if (!algo || !*algo)
return (*ops);
for (; *ops; ops++) {
if (strcmp(algo, (*ops)->name) == 0)
break;
}
ASSERT3P(ops, !=, NULL);
return (*ops);
}

8
module/zfs/zio_checksum.c
View File

@ -165,10 +165,10 @@ zio_checksum_info_t zio_checksum_table[ZIO_CHECKSUM_FUNCTIONS] = {
{{NULL, NULL}, NULL, NULL, 0, "on"},
{{abd_checksum_off, abd_checksum_off},
NULL, NULL, 0, "off"},
{{abd_checksum_SHA256, abd_checksum_SHA256},
{{abd_checksum_sha256, abd_checksum_sha256},
NULL, NULL, ZCHECKSUM_FLAG_METADATA | ZCHECKSUM_FLAG_EMBEDDED,
"label"},
{{abd_checksum_SHA256, abd_checksum_SHA256},
{{abd_checksum_sha256, abd_checksum_sha256},
NULL, NULL, ZCHECKSUM_FLAG_METADATA | ZCHECKSUM_FLAG_EMBEDDED,
"gang_header"},
{{abd_fletcher_2_native, abd_fletcher_2_byteswap},
@ -177,14 +177,14 @@ zio_checksum_info_t zio_checksum_table[ZIO_CHECKSUM_FUNCTIONS] = {
NULL, NULL, 0, "fletcher2"},
{{abd_fletcher_4_native, abd_fletcher_4_byteswap},
NULL, NULL, ZCHECKSUM_FLAG_METADATA, "fletcher4"},
{{abd_checksum_SHA256, abd_checksum_SHA256},
{{abd_checksum_sha256, abd_checksum_sha256},
NULL, NULL, ZCHECKSUM_FLAG_METADATA | ZCHECKSUM_FLAG_DEDUP |
ZCHECKSUM_FLAG_NOPWRITE, "sha256"},
{{abd_fletcher_4_native, abd_fletcher_4_byteswap},
NULL, NULL, ZCHECKSUM_FLAG_EMBEDDED, "zilog2"},
{{abd_checksum_off, abd_checksum_off},
NULL, NULL, 0, "noparity"},
{{abd_checksum_SHA512_native, abd_checksum_SHA512_byteswap},
{{abd_checksum_sha512_native, abd_checksum_sha512_byteswap},
NULL, NULL, ZCHECKSUM_FLAG_METADATA | ZCHECKSUM_FLAG_DEDUP |
ZCHECKSUM_FLAG_NOPWRITE, "sha512"},
{{abd_checksum_skein_native, abd_checksum_skein_byteswap},

34
tests/zfs-tests/cmd/checksum/sha2_test.c
View File

@ -33,11 +33,11 @@
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <sys/time.h>
#define _SHA2_IMPL
#include <sys/sha2.h>
#include <sys/stdtypes.h>
#include <sys/zfs_impl.h>
/*
* Test messages from:
@ -174,9 +174,19 @@ main(int argc, char *argv[])
boolean_t failed = B_FALSE;
uint64_t cpu_mhz = 0;
const zfs_impl_t *sha256 = zfs_impl_get_ops("sha256");
const zfs_impl_t *sha512 = zfs_impl_get_ops("sha512");
uint32_t id;
if (argc == 2)
cpu_mhz = atoi(argv[1]);
if (!sha256)
return (1);
if (!sha512)
return (1);
#define SHA2_ALGO_TEST(_m, mode, diglen, testdigest) \
do { \
SHA2_CTX ctx; \
@ -194,7 +204,7 @@ main(int argc, char *argv[])
} \
} while (0)
#define SHA2_PERF_TEST(mode, diglen) \
#define SHA2_PERF_TEST(mode, diglen, name) \
do { \
SHA2_CTX ctx; \
uint8_t digest[diglen / 8]; \
@ -216,8 +226,8 @@ main(int argc, char *argv[])
cpb = (cpu_mhz * 1e6 * ((double)delta / \
1000000)) / (8192 * 128 * 1024); \
} \
(void) printf("SHA%-9s%llu us (%.02f CPB)\n", #mode, \
(u_longlong_t)delta, cpb); \
(void) printf("sha%s-%-9s%7llu us (%.02f CPB)\n", #mode,\
name, (u_longlong_t)delta, cpb); \
} while (0)
(void) printf("Running algorithm correctness tests:\n");
@ -237,8 +247,18 @@ main(int argc, char *argv[])
(void) printf("Running performance tests (hashing 1024 MiB of "
"data):\n");
SHA2_PERF_TEST(256, 256);
SHA2_PERF_TEST(512, 512);
for (id = 0; id < sha256->getcnt(); id++) {
sha256->setid(id);
const char *name = sha256->getname();
SHA2_PERF_TEST(256, 256, name);
}
for (id = 0; id < sha512->getcnt(); id++) {
sha512->setid(id);
const char *name = sha512->getname();
SHA2_PERF_TEST(512, 512, name);
}
return (0);
}