From 14d073e2988ec98109311930732ad1ea78173877 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 16 Jun 2010 09:47:25 -0700 Subject: [PATCH 1/6] Add linux-sha2 topic branch for portability The upstream ZFS code has correctly moved to a faster native sha2 implementation. Unfortunately, under Linux that's going to be a little problematic so we revert the code to the more portable version contained in earlier ZFS releases. Using the native sha2 implementation in Linux is possible but the API is slightly different in kernel version user space depending on which libraries are used. Ideally, we need a fast implementation of SHA256 which builds as part of ZFS this shouldn't be that hard to do but it will take some effort. --- .topdeps | 3 +- .topmsg | 24 +++----- lib/libzfs/libzfs_sendrecv.c | 10 ++-- module/zfs/sha256.c | 113 +++++++++++++++++++++++++++++------ 4 files changed, 109 insertions(+), 41 deletions(-) diff --git a/.topdeps b/.topdeps index ff8849266b..7f16cbcdd5 100644 --- a/.topdeps +++ b/.topdeps @@ -1,2 +1 @@ -gcc-branch -fix-branch +zfs-branch diff --git a/.topmsg b/.topmsg index 03967cdff5..744678290c 100644 --- a/.topmsg +++ b/.topmsg @@ -1,19 +1,13 @@ From: Brian Behlendorf -Subject: [PATCH] zfs branch +Subject: [PATCH] linux-sha2 -Merged result of all changes which are relevant to both Solaris -and Linux builds of the ZFS code. These are changes where there -is a reasonable chance they will be accepted upstream. - -Additionally, since this is effectively the root of the linux -ZFS tree the core linux build system is added here. This -includes autogen.sh, configure.ac, m4 macros, some scripts/*, -and makefiles for all the core ZFS components. Linux-only -features which require tweaks to the build system should appear -on the relevant topic branches. All autotools products which -result from autogen.sh are commited to the linux-configure-branch. - -This branch also contains the META, ChangeLog, AUTHORS, TODO, -and README, files. +The upstream ZFS code has correctly moved to a faster native sha2 +implementation. Unfortunately, under Linux that's going to be a little +problematic so we revert the code to the more portable version contained +in earlier ZFS releases. Using the native sha2 implementation in Linux +is possible but the API is slightly different in kernel version user +space depending on which libraries are used. Ideally, we need a fast +implementation of SHA256 which builds as part of ZFS this shouldn't be +that hard to do but it will take some effort. Signed-off-by: Brian Behlendorf diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index e018673058..e89c51f295 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -43,7 +43,6 @@ #include "zfs_prop.h" #include "zfs_fletcher.h" #include "libzfs_impl.h" -#include #include #include @@ -336,12 +335,11 @@ cksummer(void *arg) if (ZIO_CHECKSUM_EQUAL(drrw->drr_key.ddk_cksum, zero_cksum) || !DRR_IS_DEDUP_CAPABLE(drrw->drr_checksumflags)) { - SHA256_CTX ctx; - zio_cksum_t tmpsha256; + zio_cksum_t tmpsha256; + + zio_checksum_SHA256(buf, + drrw->drr_length, &tmpsha256); - SHA256Init(&ctx); - SHA256Update(&ctx, buf, drrw->drr_length); - SHA256Final(&tmpsha256, &ctx); drrw->drr_key.ddk_cksum.zc_word[0] = BE_64(tmpsha256.zc_word[0]); drrw->drr_key.ddk_cksum.zc_word[1] = diff --git a/module/zfs/sha256.c b/module/zfs/sha256.c index f515be6bb3..cf9dd8fcba 100644 --- a/module/zfs/sha256.c +++ b/module/zfs/sha256.c @@ -19,32 +19,109 @@ * CDDL HEADER END */ /* - * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ + #include #include -#include +#include + +/* + * SHA-256 checksum, as specified in FIPS 180-3, available at: + * http://csrc.nist.gov/publications/PubsFIPS.html + * + * This is a very compact implementation of SHA-256. + * It is designed to be simple and portable, not to be fast. + */ + +/* + * The literal definitions of Ch() and Maj() according to FIPS 180-3 are: + * + * Ch(x, y, z) (x & y) ^ (~x & z) + * Maj(x, y, z) (x & y) ^ (x & z) ^ (y & z) + * + * We use equivalent logical reductions here that require one less op. + */ +#define Ch(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) +#define Maj(x, y, z) (((x) & (y)) ^ ((z) & ((x) ^ (y)))) +#define Rot32(x, s) (((x) >> s) | ((x) << (32 - s))) +#define SIGMA0(x) (Rot32(x, 2) ^ Rot32(x, 13) ^ Rot32(x, 22)) +#define SIGMA1(x) (Rot32(x, 6) ^ Rot32(x, 11) ^ Rot32(x, 25)) +#define sigma0(x) (Rot32(x, 7) ^ Rot32(x, 18) ^ ((x) >> 3)) +#define sigma1(x) (Rot32(x, 17) ^ Rot32(x, 19) ^ ((x) >> 10)) + +static const uint32_t SHA256_K[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +}; + +static void +SHA256Transform(uint32_t *H, const uint8_t *cp) +{ + uint32_t a, b, c, d, e, f, g, h, t, T1, T2, W[64]; + + for (t = 0; t < 16; t++, cp += 4) + W[t] = (cp[0] << 24) | (cp[1] << 16) | (cp[2] << 8) | cp[3]; + + for (t = 16; t < 64; t++) + W[t] = sigma1(W[t - 2]) + W[t - 7] + + sigma0(W[t - 15]) + W[t - 16]; + + a = H[0]; b = H[1]; c = H[2]; d = H[3]; + e = H[4]; f = H[5]; g = H[6]; h = H[7]; + + for (t = 0; t < 64; t++) { + T1 = h + SIGMA1(e) + Ch(e, f, g) + SHA256_K[t] + W[t]; + T2 = SIGMA0(a) + Maj(a, b, c); + h = g; g = f; f = e; e = d + T1; + d = c; c = b; b = a; a = T1 + T2; + } + + H[0] += a; H[1] += b; H[2] += c; H[3] += d; + H[4] += e; H[5] += f; H[6] += g; H[7] += h; +} void zio_checksum_SHA256(const void *buf, uint64_t size, zio_cksum_t *zcp) { - SHA2_CTX ctx; - zio_cksum_t tmp; + uint32_t H[8] = { 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19 }; + uint8_t pad[128]; + int i, padsize; - SHA2Init(SHA256, &ctx); - SHA2Update(&ctx, buf, size); - SHA2Final(&tmp, &ctx); + for (i = 0; i < (size & ~63ULL); i += 64) + SHA256Transform(H, (uint8_t *)buf + i); - /* - * A prior implementation of this function had a - * private SHA256 implementation always wrote things out in - * Big Endian and there wasn't a byteswap variant of it. - * To preseve on disk compatibility we need to force that - * behaviour. - */ - zcp->zc_word[0] = BE_64(tmp.zc_word[0]); - zcp->zc_word[1] = BE_64(tmp.zc_word[1]); - zcp->zc_word[2] = BE_64(tmp.zc_word[2]); - zcp->zc_word[3] = BE_64(tmp.zc_word[3]); + for (padsize = 0; i < size; i++) + pad[padsize++] = *((uint8_t *)buf + i); + + for (pad[padsize++] = 0x80; (padsize & 63) != 56; padsize++) + pad[padsize] = 0; + + for (i = 56; i >= 0; i -= 8) + pad[padsize++] = (size << 3) >> i; + + for (i = 0; i < padsize; i += 64) + SHA256Transform(H, pad + i); + + ZIO_SET_CHECKSUM(zcp, + (uint64_t)H[0] << 32 | H[1], + (uint64_t)H[2] << 32 | H[3], + (uint64_t)H[4] << 32 | H[5], + (uint64_t)H[6] << 32 | H[7]); } From 034e0a932ba6bb5c43f09f2730b19352d6d7b899 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 16 Jun 2010 09:48:07 -0700 Subject: [PATCH 2/6] New TopGit dependency: linux-sha2 --- .topdeps | 1 + 1 file changed, 1 insertion(+) diff --git a/.topdeps b/.topdeps index e4d01fea4a..53820ce794 100644 --- a/.topdeps +++ b/.topdeps @@ -13,3 +13,4 @@ linux-kernel-device linux-events linux-debug-zerocopy linux-user-util +linux-sha2 From 8c7748d8e0290f8b095f964086a67bea07893ce4 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 16 Jun 2010 09:55:34 -0700 Subject: [PATCH 3/6] Add linux-topology topic branch Solaris recently introduced the idea of drive topology because where a drive is located does matter. I have already handled this with udev/blkid integration under Linux so I'm hopeful this case can simply be removed but for now I've just stubbed out what is needed in libspl and commented out the rest here. --- .topdeps | 3 +-- .topmsg | 21 ++++++--------------- lib/libzfs/include/libzfs_impl.h | 4 ++++ lib/libzfs/libzfs_dataset.c | 1 + lib/libzfs/libzfs_fru.c | 17 +++++++++++++++++ 5 files changed, 29 insertions(+), 17 deletions(-) diff --git a/.topdeps b/.topdeps index ff8849266b..7f16cbcdd5 100644 --- a/.topdeps +++ b/.topdeps @@ -1,2 +1 @@ -gcc-branch -fix-branch +zfs-branch diff --git a/.topmsg b/.topmsg index 03967cdff5..86d40a0843 100644 --- a/.topmsg +++ b/.topmsg @@ -1,19 +1,10 @@ From: Brian Behlendorf -Subject: [PATCH] zfs branch +Subject: [PATCH] linux topology -Merged result of all changes which are relevant to both Solaris -and Linux builds of the ZFS code. These are changes where there -is a reasonable chance they will be accepted upstream. - -Additionally, since this is effectively the root of the linux -ZFS tree the core linux build system is added here. This -includes autogen.sh, configure.ac, m4 macros, some scripts/*, -and makefiles for all the core ZFS components. Linux-only -features which require tweaks to the build system should appear -on the relevant topic branches. All autotools products which -result from autogen.sh are commited to the linux-configure-branch. - -This branch also contains the META, ChangeLog, AUTHORS, TODO, -and README, files. +Solaris recently introduced the idea of drive topology because +where a drive is located does matter. I have already handled +this with udev/blkid integration under Linux so I'm hopeful +this case can simply be removed but for now I've just stubbed +out what is needed in libspl and commented out the rest here. Signed-off-by: Brian Behlendorf diff --git a/lib/libzfs/include/libzfs_impl.h b/lib/libzfs/include/libzfs_impl.h index 89c48c1c03..3f540e37d2 100644 --- a/lib/libzfs/include/libzfs_impl.h +++ b/lib/libzfs/include/libzfs_impl.h @@ -37,7 +37,9 @@ #include #include +#if defined(HAVE_LIBTOPO) #include +#endif /* HAVE_LIBTOPO */ #ifdef __cplusplus extern "C" { @@ -74,9 +76,11 @@ struct libzfs_handle { boolean_t libzfs_mnttab_enable; avl_tree_t libzfs_mnttab_cache; int libzfs_pool_iter; +#if defined(HAVE_LIBTOPO) topo_hdl_t *libzfs_topo_hdl; libzfs_fru_t **libzfs_fru_hash; libzfs_fru_t *libzfs_fru_list; +#endif /* HAVE_LIBTOPO */ char libzfs_chassis_id[256]; }; diff --git a/lib/libzfs/libzfs_dataset.c b/lib/libzfs/libzfs_dataset.c index 17377013cf..40164854e6 100644 --- a/lib/libzfs/libzfs_dataset.c +++ b/lib/libzfs/libzfs_dataset.c @@ -26,6 +26,7 @@ #include #include +#include #include #include #include diff --git a/lib/libzfs/libzfs_fru.c b/lib/libzfs/libzfs_fru.c index 788fa2cfb7..78f2f9c371 100644 --- a/lib/libzfs/libzfs_fru.c +++ b/lib/libzfs/libzfs_fru.c @@ -34,6 +34,8 @@ #include +#if defined(HAVE_LIBTOPO) + #include #include #include @@ -450,3 +452,18 @@ libzfs_fru_clear(libzfs_handle_t *hdl, boolean_t final) ZFS_FRU_HASH_SIZE * sizeof (void *)); } } + +#else /* HAVE_LIBTOPO */ + +/* + * Clear memory associated with the FRU hash. + */ +void +libzfs_fru_clear(libzfs_handle_t *hdl, boolean_t final) +{ + return; +} + +#endif /* HAVE_LIBTOPO */ + + From c23ab08994a46d7f16842303b0cc03e49cdb054d Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 16 Jun 2010 09:56:09 -0700 Subject: [PATCH 4/6] New TopGit dependency: linux-topology --- .topdeps | 1 + 1 file changed, 1 insertion(+) diff --git a/.topdeps b/.topdeps index 53820ce794..7caad8edc4 100644 --- a/.topdeps +++ b/.topdeps @@ -14,3 +14,4 @@ linux-events linux-debug-zerocopy linux-user-util linux-sha2 +linux-topology From 6104e13be6ccd83fad00a130bc5fa952673e9781 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 16 Jun 2010 09:59:39 -0700 Subject: [PATCH 5/6] Add linux-spa-thread topic branch Disable the spa thread under Linux until it can be implemented. --- .topdeps | 3 +-- .topmsg | 17 ++--------------- module/zfs/spa.c | 4 +++- 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/.topdeps b/.topdeps index ff8849266b..7f16cbcdd5 100644 --- a/.topdeps +++ b/.topdeps @@ -1,2 +1 @@ -gcc-branch -fix-branch +zfs-branch diff --git a/.topmsg b/.topmsg index 03967cdff5..26d41d79e2 100644 --- a/.topmsg +++ b/.topmsg @@ -1,19 +1,6 @@ From: Brian Behlendorf -Subject: [PATCH] zfs branch +Subject: [PATCH] linux spa thread -Merged result of all changes which are relevant to both Solaris -and Linux builds of the ZFS code. These are changes where there -is a reasonable chance they will be accepted upstream. - -Additionally, since this is effectively the root of the linux -ZFS tree the core linux build system is added here. This -includes autogen.sh, configure.ac, m4 macros, some scripts/*, -and makefiles for all the core ZFS components. Linux-only -features which require tweaks to the build system should appear -on the relevant topic branches. All autotools products which -result from autogen.sh are commited to the linux-configure-branch. - -This branch also contains the META, ChangeLog, AUTHORS, TODO, -and README, files. +Disable the spa thread under Linux until it can be implemented. Signed-off-by: Brian Behlendorf diff --git a/module/zfs/spa.c b/module/zfs/spa.c index 6b90cc2309..376a4b7ae4 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -669,7 +669,7 @@ spa_create_zio_taskqs(spa_t *spa) } } -#ifdef _KERNEL +#if defined(_KERNEL) && defined(HAVE_SPA_THREAD) static void spa_thread(void *arg) { @@ -759,6 +759,7 @@ spa_activate(spa_t *spa, int mode) ASSERT(spa->spa_proc == &p0); spa->spa_did = 0; +#ifdef HAVE_SPA_THREAD /* Only create a process if we're going to be around a while. */ if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) { if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri, @@ -779,6 +780,7 @@ spa_activate(spa_t *spa, int mode) #endif } } +#endif /* HAVE_SPA_THREAD */ mutex_exit(&spa->spa_proc_lock); /* If we didn't create a process, we need to create our taskqs. */ From 991e3fc98fc57d09d1c72fcec12554a9f61a171c Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 16 Jun 2010 10:00:02 -0700 Subject: [PATCH 6/6] New TopGit dependency: linux-spa-thread --- .topdeps | 1 + 1 file changed, 1 insertion(+) diff --git a/.topdeps b/.topdeps index 7caad8edc4..7eb36b2d65 100644 --- a/.topdeps +++ b/.topdeps @@ -15,3 +15,4 @@ linux-debug-zerocopy linux-user-util linux-sha2 linux-topology +linux-spa-thread