Illumos Crypto Port module added to enable native encryption in zfs

A port of the Illumos Crypto Framework to a Linux kernel module (found
in module/icp). This is needed to do the actual encryption work. We cannot
use the Linux kernel's built in crypto api because it is only exported to
GPL-licensed modules. Having the ICP also means the crypto code can run on
any of the other kernels under OpenZFS. I ended up porting over most of the
internals of the framework, which means that porting over other API calls (if
we need them) should be fairly easy. Specifically, I have ported over the API
functions related to encryption, digests, macs, and crypto templates. The ICP
is able to use assembly-accelerated encryption on amd64 machines and AES-NI
instructions on Intel chips that support it. There are place-holder
directories for similar assembly optimizations for other architectures
(although they have not been written).

Signed-off-by: Tom Caputi <tcaputi@datto.com>
Signed-off-by: Tony Hutter <hutter2@llnl.gov>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Issue #4329

module/icp/algs/aes/aes_modes.c

@@ -0,0 +1,135 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <aes/aes_impl.h>
+
+/* Copy a 16-byte AES block from "in" to "out" */
+void
+aes_copy_block(uint8_t *in, uint8_t *out)
+{
+	if (IS_P2ALIGNED2(in, out, sizeof (uint32_t))) {
+		/* LINTED: pointer alignment */
+		*(uint32_t *)&out[0] = *(uint32_t *)&in[0];
+		/* LINTED: pointer alignment */
+		*(uint32_t *)&out[4] = *(uint32_t *)&in[4];
+		/* LINTED: pointer alignment */
+		*(uint32_t *)&out[8] = *(uint32_t *)&in[8];
+		/* LINTED: pointer alignment */
+		*(uint32_t *)&out[12] = *(uint32_t *)&in[12];
+	} else {
+		AES_COPY_BLOCK(in, out);
+	}
+}
+
+
+/* XOR a 16-byte AES block of data into dst */
+void
+aes_xor_block(uint8_t *data, uint8_t *dst)
+{
+	if (IS_P2ALIGNED2(dst, data, sizeof (uint32_t))) {
+		/* LINTED: pointer alignment */
+		*(uint32_t *)&dst[0] ^= *(uint32_t *)&data[0];
+		/* LINTED: pointer alignment */
+		*(uint32_t *)&dst[4] ^= *(uint32_t *)&data[4];
+		/* LINTED: pointer alignment */
+		*(uint32_t *)&dst[8] ^= *(uint32_t *)&data[8];
+		/* LINTED: pointer alignment */
+		*(uint32_t *)&dst[12] ^= *(uint32_t *)&data[12];
+	} else {
+		AES_XOR_BLOCK(data, dst);
+	}
+}
+
+
+/*
+ * Encrypt multiple blocks of data according to mode.
+ */
+int
+aes_encrypt_contiguous_blocks(void *ctx, char *data, size_t length,
+    crypto_data_t *out)
+{
+	aes_ctx_t *aes_ctx = ctx;
+	int rv;
+
+	if (aes_ctx->ac_flags & CTR_MODE) {
+		rv = ctr_mode_contiguous_blocks(ctx, data, length, out,
+		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+	} else if (aes_ctx->ac_flags & CCM_MODE) {
+		rv = ccm_mode_encrypt_contiguous_blocks(ctx, data, length,
+		    out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
+		    aes_xor_block);
+	} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
+		rv = gcm_mode_encrypt_contiguous_blocks(ctx, data, length,
+		    out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
+		    aes_xor_block);
+	} else if (aes_ctx->ac_flags & CBC_MODE) {
+		rv = cbc_encrypt_contiguous_blocks(ctx,
+		    data, length, out, AES_BLOCK_LEN, aes_encrypt_block,
+		    aes_copy_block, aes_xor_block);
+	} else {
+		rv = ecb_cipher_contiguous_blocks(ctx, data, length, out,
+		    AES_BLOCK_LEN, aes_encrypt_block);
+	}
+	return (rv);
+}
+
+
+/*
+ * Decrypt multiple blocks of data according to mode.
+ */
+int
+aes_decrypt_contiguous_blocks(void *ctx, char *data, size_t length,
+    crypto_data_t *out)
+{
+	aes_ctx_t *aes_ctx = ctx;
+	int rv;
+
+	if (aes_ctx->ac_flags & CTR_MODE) {
+		rv = ctr_mode_contiguous_blocks(ctx, data, length, out,
+		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+		if (rv == CRYPTO_DATA_LEN_RANGE)
+			rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE;
+	} else if (aes_ctx->ac_flags & CCM_MODE) {
+		rv = ccm_mode_decrypt_contiguous_blocks(ctx, data, length,
+		    out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
+		    aes_xor_block);
+	} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
+		rv = gcm_mode_decrypt_contiguous_blocks(ctx, data, length,
+		    out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
+		    aes_xor_block);
+	} else if (aes_ctx->ac_flags & CBC_MODE) {
+		rv = cbc_decrypt_contiguous_blocks(ctx, data, length, out,
+		    AES_BLOCK_LEN, aes_decrypt_block, aes_copy_block,
+		    aes_xor_block);
+	} else {
+		rv = ecb_cipher_contiguous_blocks(ctx, data, length, out,
+		    AES_BLOCK_LEN, aes_decrypt_block);
+		if (rv == CRYPTO_DATA_LEN_RANGE)
+			rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE;
+	}
+	return (rv);
+}

module/icp/algs/modes/cbc.c

@@ -0,0 +1,305 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+
+/*
+ * Algorithm independent CBC functions.
+ */
+int
+cbc_encrypt_contiguous_blocks(cbc_ctx_t *ctx, char *data, size_t length,
+    crypto_data_t *out, size_t block_size,
+    int (*encrypt)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	size_t remainder = length;
+	size_t need = 0;
+	uint8_t *datap = (uint8_t *)data;
+	uint8_t *blockp;
+	uint8_t *lastp;
+	void *iov_or_mp;
+	offset_t offset;
+	uint8_t *out_data_1;
+	uint8_t *out_data_2;
+	size_t out_data_1_len;
+
+	if (length + ctx->cbc_remainder_len < block_size) {
+		/* accumulate bytes here and return */
+		bcopy(datap,
+		    (uint8_t *)ctx->cbc_remainder + ctx->cbc_remainder_len,
+		    length);
+		ctx->cbc_remainder_len += length;
+		ctx->cbc_copy_to = datap;
+		return (CRYPTO_SUCCESS);
+	}
+
+	lastp = (uint8_t *)ctx->cbc_iv;
+	if (out != NULL)
+		crypto_init_ptrs(out, &iov_or_mp, &offset);
+
+	do {
+		/* Unprocessed data from last call. */
+		if (ctx->cbc_remainder_len > 0) {
+			need = block_size - ctx->cbc_remainder_len;
+
+			if (need > remainder)
+				return (CRYPTO_DATA_LEN_RANGE);
+
+			bcopy(datap, &((uint8_t *)ctx->cbc_remainder)
+			    [ctx->cbc_remainder_len], need);
+
+			blockp = (uint8_t *)ctx->cbc_remainder;
+		} else {
+			blockp = datap;
+		}
+
+		if (out == NULL) {
+			/*
+			 * XOR the previous cipher block or IV with the
+			 * current clear block.
+			 */
+			xor_block(lastp, blockp);
+			encrypt(ctx->cbc_keysched, blockp, blockp);
+
+			ctx->cbc_lastp = blockp;
+			lastp = blockp;
+
+			if (ctx->cbc_remainder_len > 0) {
+				bcopy(blockp, ctx->cbc_copy_to,
+				    ctx->cbc_remainder_len);
+				bcopy(blockp + ctx->cbc_remainder_len, datap,
+				    need);
+			}
+		} else {
+			/*
+			 * XOR the previous cipher block or IV with the
+			 * current clear block.
+			 */
+			xor_block(blockp, lastp);
+			encrypt(ctx->cbc_keysched, lastp, lastp);
+			crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+			    &out_data_1_len, &out_data_2, block_size);
+
+			/* copy block to where it belongs */
+			if (out_data_1_len == block_size) {
+				copy_block(lastp, out_data_1);
+			} else {
+				bcopy(lastp, out_data_1, out_data_1_len);
+				if (out_data_2 != NULL) {
+					bcopy(lastp + out_data_1_len,
+					    out_data_2,
+					    block_size - out_data_1_len);
+				}
+			}
+			/* update offset */
+			out->cd_offset += block_size;
+		}
+
+		/* Update pointer to next block of data to be processed. */
+		if (ctx->cbc_remainder_len != 0) {
+			datap += need;
+			ctx->cbc_remainder_len = 0;
+		} else {
+			datap += block_size;
+		}
+
+		remainder = (size_t)&data[length] - (size_t)datap;
+
+		/* Incomplete last block. */
+		if (remainder > 0 && remainder < block_size) {
+			bcopy(datap, ctx->cbc_remainder, remainder);
+			ctx->cbc_remainder_len = remainder;
+			ctx->cbc_copy_to = datap;
+			goto out;
+		}
+		ctx->cbc_copy_to = NULL;
+
+	} while (remainder > 0);
+
+out:
+	/*
+	 * Save the last encrypted block in the context.
+	 */
+	if (ctx->cbc_lastp != NULL) {
+		copy_block((uint8_t *)ctx->cbc_lastp, (uint8_t *)ctx->cbc_iv);
+		ctx->cbc_lastp = (uint8_t *)ctx->cbc_iv;
+	}
+
+	return (CRYPTO_SUCCESS);
+}
+
+#define	OTHER(a, ctx) \
+	(((a) == (ctx)->cbc_lastblock) ? (ctx)->cbc_iv : (ctx)->cbc_lastblock)
+
+/* ARGSUSED */
+int
+cbc_decrypt_contiguous_blocks(cbc_ctx_t *ctx, char *data, size_t length,
+    crypto_data_t *out, size_t block_size,
+    int (*decrypt)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	size_t remainder = length;
+	size_t need = 0;
+	uint8_t *datap = (uint8_t *)data;
+	uint8_t *blockp;
+	uint8_t *lastp;
+	void *iov_or_mp;
+	offset_t offset;
+	uint8_t *out_data_1;
+	uint8_t *out_data_2;
+	size_t out_data_1_len;
+
+	if (length + ctx->cbc_remainder_len < block_size) {
+		/* accumulate bytes here and return */
+		bcopy(datap,
+		    (uint8_t *)ctx->cbc_remainder + ctx->cbc_remainder_len,
+		    length);
+		ctx->cbc_remainder_len += length;
+		ctx->cbc_copy_to = datap;
+		return (CRYPTO_SUCCESS);
+	}
+
+	lastp = ctx->cbc_lastp;
+	if (out != NULL)
+		crypto_init_ptrs(out, &iov_or_mp, &offset);
+
+	do {
+		/* Unprocessed data from last call. */
+		if (ctx->cbc_remainder_len > 0) {
+			need = block_size - ctx->cbc_remainder_len;
+
+			if (need > remainder)
+				return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
+
+			bcopy(datap, &((uint8_t *)ctx->cbc_remainder)
+			    [ctx->cbc_remainder_len], need);
+
+			blockp = (uint8_t *)ctx->cbc_remainder;
+		} else {
+			blockp = datap;
+		}
+
+		/* LINTED: pointer alignment */
+		copy_block(blockp, (uint8_t *)OTHER((uint64_t *)lastp, ctx));
+
+		if (out != NULL) {
+			decrypt(ctx->cbc_keysched, blockp,
+			    (uint8_t *)ctx->cbc_remainder);
+			blockp = (uint8_t *)ctx->cbc_remainder;
+		} else {
+			decrypt(ctx->cbc_keysched, blockp, blockp);
+		}
+
+		/*
+		 * XOR the previous cipher block or IV with the
+		 * currently decrypted block.
+		 */
+		xor_block(lastp, blockp);
+
+		/* LINTED: pointer alignment */
+		lastp = (uint8_t *)OTHER((uint64_t *)lastp, ctx);
+
+		if (out != NULL) {
+			crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+			    &out_data_1_len, &out_data_2, block_size);
+
+			bcopy(blockp, out_data_1, out_data_1_len);
+			if (out_data_2 != NULL) {
+				bcopy(blockp + out_data_1_len, out_data_2,
+				    block_size - out_data_1_len);
+			}
+
+			/* update offset */
+			out->cd_offset += block_size;
+
+		} else if (ctx->cbc_remainder_len > 0) {
+			/* copy temporary block to where it belongs */
+			bcopy(blockp, ctx->cbc_copy_to, ctx->cbc_remainder_len);
+			bcopy(blockp + ctx->cbc_remainder_len, datap, need);
+		}
+
+		/* Update pointer to next block of data to be processed. */
+		if (ctx->cbc_remainder_len != 0) {
+			datap += need;
+			ctx->cbc_remainder_len = 0;
+		} else {
+			datap += block_size;
+		}
+
+		remainder = (size_t)&data[length] - (size_t)datap;
+
+		/* Incomplete last block. */
+		if (remainder > 0 && remainder < block_size) {
+			bcopy(datap, ctx->cbc_remainder, remainder);
+			ctx->cbc_remainder_len = remainder;
+			ctx->cbc_lastp = lastp;
+			ctx->cbc_copy_to = datap;
+			return (CRYPTO_SUCCESS);
+		}
+		ctx->cbc_copy_to = NULL;
+
+	} while (remainder > 0);
+
+	ctx->cbc_lastp = lastp;
+	return (CRYPTO_SUCCESS);
+}
+
+int
+cbc_init_ctx(cbc_ctx_t *cbc_ctx, char *param, size_t param_len,
+    size_t block_size, void (*copy_block)(uint8_t *, uint64_t *))
+{
+	/*
+	 * Copy IV into context.
+	 *
+	 * If cm_param == NULL then the IV comes from the
+	 * cd_miscdata field in the crypto_data structure.
+	 */
+	if (param != NULL) {
+		ASSERT(param_len == block_size);
+		copy_block((uchar_t *)param, cbc_ctx->cbc_iv);
+	}
+
+	cbc_ctx->cbc_lastp = (uint8_t *)&cbc_ctx->cbc_iv[0];
+	cbc_ctx->cbc_flags |= CBC_MODE;
+	return (CRYPTO_SUCCESS);
+}
+
+/* ARGSUSED */
+void *
+cbc_alloc_ctx(int kmflag)
+{
+	cbc_ctx_t *cbc_ctx;
+
+	if ((cbc_ctx = kmem_zalloc(sizeof (cbc_ctx_t), kmflag)) == NULL)
+		return (NULL);
+
+	cbc_ctx->cbc_flags = CBC_MODE;
+	return (cbc_ctx);
+}

module/icp/algs/modes/ccm.c

@@ -0,0 +1,920 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+
+#if defined(__i386) || defined(__amd64)
+#include <sys/byteorder.h>
+#define	UNALIGNED_POINTERS_PERMITTED
+#endif
+
+/*
+ * Encrypt multiple blocks of data in CCM mode.  Decrypt for CCM mode
+ * is done in another function.
+ */
+int
+ccm_mode_encrypt_contiguous_blocks(ccm_ctx_t *ctx, char *data, size_t length,
+    crypto_data_t *out, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	size_t remainder = length;
+	size_t need = 0;
+	uint8_t *datap = (uint8_t *)data;
+	uint8_t *blockp;
+	uint8_t *lastp;
+	void *iov_or_mp;
+	offset_t offset;
+	uint8_t *out_data_1;
+	uint8_t *out_data_2;
+	size_t out_data_1_len;
+	uint64_t counter;
+	uint8_t *mac_buf;
+
+	if (length + ctx->ccm_remainder_len < block_size) {
+		/* accumulate bytes here and return */
+		bcopy(datap,
+		    (uint8_t *)ctx->ccm_remainder + ctx->ccm_remainder_len,
+		    length);
+		ctx->ccm_remainder_len += length;
+		ctx->ccm_copy_to = datap;
+		return (CRYPTO_SUCCESS);
+	}
+
+	lastp = (uint8_t *)ctx->ccm_cb;
+	if (out != NULL)
+		crypto_init_ptrs(out, &iov_or_mp, &offset);
+
+	mac_buf = (uint8_t *)ctx->ccm_mac_buf;
+
+	do {
+		/* Unprocessed data from last call. */
+		if (ctx->ccm_remainder_len > 0) {
+			need = block_size - ctx->ccm_remainder_len;
+
+			if (need > remainder)
+				return (CRYPTO_DATA_LEN_RANGE);
+
+			bcopy(datap, &((uint8_t *)ctx->ccm_remainder)
+			    [ctx->ccm_remainder_len], need);
+
+			blockp = (uint8_t *)ctx->ccm_remainder;
+		} else {
+			blockp = datap;
+		}
+
+		/*
+		 * do CBC MAC
+		 *
+		 * XOR the previous cipher block current clear block.
+		 * mac_buf always contain previous cipher block.
+		 */
+		xor_block(blockp, mac_buf);
+		encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
+
+		/* ccm_cb is the counter block */
+		encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb,
+		    (uint8_t *)ctx->ccm_tmp);
+
+		lastp = (uint8_t *)ctx->ccm_tmp;
+
+		/*
+		 * Increment counter. Counter bits are confined
+		 * to the bottom 64 bits of the counter block.
+		 */
+#ifdef _LITTLE_ENDIAN
+		counter = ntohll(ctx->ccm_cb[1] & ctx->ccm_counter_mask);
+		counter = htonll(counter + 1);
+#else
+		counter = ctx->ccm_cb[1] & ctx->ccm_counter_mask;
+		counter++;
+#endif	/* _LITTLE_ENDIAN */
+		counter &= ctx->ccm_counter_mask;
+		ctx->ccm_cb[1] =
+		    (ctx->ccm_cb[1] & ~(ctx->ccm_counter_mask)) | counter;
+
+		/*
+		 * XOR encrypted counter block with the current clear block.
+		 */
+		xor_block(blockp, lastp);
+
+		ctx->ccm_processed_data_len += block_size;
+
+		if (out == NULL) {
+			if (ctx->ccm_remainder_len > 0) {
+				bcopy(blockp, ctx->ccm_copy_to,
+				    ctx->ccm_remainder_len);
+				bcopy(blockp + ctx->ccm_remainder_len, datap,
+				    need);
+			}
+		} else {
+			crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+			    &out_data_1_len, &out_data_2, block_size);
+
+			/* copy block to where it belongs */
+			if (out_data_1_len == block_size) {
+				copy_block(lastp, out_data_1);
+			} else {
+				bcopy(lastp, out_data_1, out_data_1_len);
+				if (out_data_2 != NULL) {
+					bcopy(lastp + out_data_1_len,
+					    out_data_2,
+					    block_size - out_data_1_len);
+				}
+			}
+			/* update offset */
+			out->cd_offset += block_size;
+		}
+
+		/* Update pointer to next block of data to be processed. */
+		if (ctx->ccm_remainder_len != 0) {
+			datap += need;
+			ctx->ccm_remainder_len = 0;
+		} else {
+			datap += block_size;
+		}
+
+		remainder = (size_t)&data[length] - (size_t)datap;
+
+		/* Incomplete last block. */
+		if (remainder > 0 && remainder < block_size) {
+			bcopy(datap, ctx->ccm_remainder, remainder);
+			ctx->ccm_remainder_len = remainder;
+			ctx->ccm_copy_to = datap;
+			goto out;
+		}
+		ctx->ccm_copy_to = NULL;
+
+	} while (remainder > 0);
+
+out:
+	return (CRYPTO_SUCCESS);
+}
+
+void
+calculate_ccm_mac(ccm_ctx_t *ctx, uint8_t *ccm_mac,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *))
+{
+	uint64_t counter;
+	uint8_t *counterp, *mac_buf;
+	int i;
+
+	mac_buf = (uint8_t *)ctx->ccm_mac_buf;
+
+	/* first counter block start with index 0 */
+	counter = 0;
+	ctx->ccm_cb[1] = (ctx->ccm_cb[1] & ~(ctx->ccm_counter_mask)) | counter;
+
+	counterp = (uint8_t *)ctx->ccm_tmp;
+	encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, counterp);
+
+	/* calculate XOR of MAC with first counter block */
+	for (i = 0; i < ctx->ccm_mac_len; i++) {
+		ccm_mac[i] = mac_buf[i] ^ counterp[i];
+	}
+}
+
+/* ARGSUSED */
+int
+ccm_encrypt_final(ccm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	uint8_t *lastp, *mac_buf, *ccm_mac_p, *macp = NULL;
+	void *iov_or_mp;
+	offset_t offset;
+	uint8_t *out_data_1;
+	uint8_t *out_data_2;
+	size_t out_data_1_len;
+	int i;
+
+	if (out->cd_length < (ctx->ccm_remainder_len + ctx->ccm_mac_len)) {
+		return (CRYPTO_DATA_LEN_RANGE);
+	}
+
+	/*
+	 * When we get here, the number of bytes of payload processed
+	 * plus whatever data remains, if any,
+	 * should be the same as the number of bytes that's being
+	 * passed in the argument during init time.
+	 */
+	if ((ctx->ccm_processed_data_len + ctx->ccm_remainder_len)
+	    != (ctx->ccm_data_len)) {
+		return (CRYPTO_DATA_LEN_RANGE);
+	}
+
+	mac_buf = (uint8_t *)ctx->ccm_mac_buf;
+
+	if (ctx->ccm_remainder_len > 0) {
+
+		/* ccm_mac_input_buf is not used for encryption */
+		macp = (uint8_t *)ctx->ccm_mac_input_buf;
+		bzero(macp, block_size);
+
+		/* copy remainder to temporary buffer */
+		bcopy(ctx->ccm_remainder, macp, ctx->ccm_remainder_len);
+
+		/* calculate the CBC MAC */
+		xor_block(macp, mac_buf);
+		encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
+
+		/* calculate the counter mode */
+		lastp = (uint8_t *)ctx->ccm_tmp;
+		encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, lastp);
+
+		/* XOR with counter block */
+		for (i = 0; i < ctx->ccm_remainder_len; i++) {
+			macp[i] ^= lastp[i];
+		}
+		ctx->ccm_processed_data_len += ctx->ccm_remainder_len;
+	}
+
+	/* Calculate the CCM MAC */
+	ccm_mac_p = (uint8_t *)ctx->ccm_tmp;
+	calculate_ccm_mac(ctx, ccm_mac_p, encrypt_block);
+
+	crypto_init_ptrs(out, &iov_or_mp, &offset);
+	crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+	    &out_data_1_len, &out_data_2,
+	    ctx->ccm_remainder_len + ctx->ccm_mac_len);
+
+	if (ctx->ccm_remainder_len > 0) {
+
+		/* copy temporary block to where it belongs */
+		if (out_data_2 == NULL) {
+			/* everything will fit in out_data_1 */
+			bcopy(macp, out_data_1, ctx->ccm_remainder_len);
+			bcopy(ccm_mac_p, out_data_1 + ctx->ccm_remainder_len,
+			    ctx->ccm_mac_len);
+		} else {
+
+			if (out_data_1_len < ctx->ccm_remainder_len) {
+
+				size_t data_2_len_used;
+
+				bcopy(macp, out_data_1, out_data_1_len);
+
+				data_2_len_used = ctx->ccm_remainder_len
+				    - out_data_1_len;
+
+				bcopy((uint8_t *)macp + out_data_1_len,
+				    out_data_2, data_2_len_used);
+				bcopy(ccm_mac_p, out_data_2 + data_2_len_used,
+				    ctx->ccm_mac_len);
+			} else {
+				bcopy(macp, out_data_1, out_data_1_len);
+				if (out_data_1_len == ctx->ccm_remainder_len) {
+					/* mac will be in out_data_2 */
+					bcopy(ccm_mac_p, out_data_2,
+					    ctx->ccm_mac_len);
+				} else {
+					size_t len_not_used = out_data_1_len -
+					    ctx->ccm_remainder_len;
+					/*
+					 * part of mac in will be in
+					 * out_data_1, part of the mac will be
+					 * in out_data_2
+					 */
+					bcopy(ccm_mac_p,
+					    out_data_1 + ctx->ccm_remainder_len,
+					    len_not_used);
+					bcopy(ccm_mac_p + len_not_used,
+					    out_data_2,
+					    ctx->ccm_mac_len - len_not_used);
+
+				}
+			}
+		}
+	} else {
+		/* copy block to where it belongs */
+		bcopy(ccm_mac_p, out_data_1, out_data_1_len);
+		if (out_data_2 != NULL) {
+			bcopy(ccm_mac_p + out_data_1_len, out_data_2,
+			    block_size - out_data_1_len);
+		}
+	}
+	out->cd_offset += ctx->ccm_remainder_len + ctx->ccm_mac_len;
+	ctx->ccm_remainder_len = 0;
+	return (CRYPTO_SUCCESS);
+}
+
+/*
+ * This will only deal with decrypting the last block of the input that
+ * might not be a multiple of block length.
+ */
+void
+ccm_decrypt_incomplete_block(ccm_ctx_t *ctx,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *))
+{
+	uint8_t *datap, *outp, *counterp;
+	int i;
+
+	datap = (uint8_t *)ctx->ccm_remainder;
+	outp = &((ctx->ccm_pt_buf)[ctx->ccm_processed_data_len]);
+
+	counterp = (uint8_t *)ctx->ccm_tmp;
+	encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, counterp);
+
+	/* XOR with counter block */
+	for (i = 0; i < ctx->ccm_remainder_len; i++) {
+		outp[i] = datap[i] ^ counterp[i];
+	}
+}
+
+/*
+ * This will decrypt the cipher text.  However, the plaintext won't be
+ * returned to the caller.  It will be returned when decrypt_final() is
+ * called if the MAC matches
+ */
+/* ARGSUSED */
+int
+ccm_mode_decrypt_contiguous_blocks(ccm_ctx_t *ctx, char *data, size_t length,
+    crypto_data_t *out, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	size_t remainder = length;
+	size_t need = 0;
+	uint8_t *datap = (uint8_t *)data;
+	uint8_t *blockp;
+	uint8_t *cbp;
+	uint64_t counter;
+	size_t pt_len, total_decrypted_len, mac_len, pm_len, pd_len;
+	uint8_t *resultp;
+
+
+	pm_len = ctx->ccm_processed_mac_len;
+
+	if (pm_len > 0) {
+		uint8_t *tmp;
+		/*
+		 * all ciphertext has been processed, just waiting for
+		 * part of the value of the mac
+		 */
+		if ((pm_len + length) > ctx->ccm_mac_len) {
+			return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
+		}
+		tmp = (uint8_t *)ctx->ccm_mac_input_buf;
+
+		bcopy(datap, tmp + pm_len, length);
+
+		ctx->ccm_processed_mac_len += length;
+		return (CRYPTO_SUCCESS);
+	}
+
+	/*
+	 * If we decrypt the given data, what total amount of data would
+	 * have been decrypted?
+	 */
+	pd_len = ctx->ccm_processed_data_len;
+	total_decrypted_len = pd_len + length + ctx->ccm_remainder_len;
+
+	if (total_decrypted_len >
+	    (ctx->ccm_data_len + ctx->ccm_mac_len)) {
+		return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
+	}
+
+	pt_len = ctx->ccm_data_len;
+
+	if (total_decrypted_len > pt_len) {
+		/*
+		 * part of the input will be the MAC, need to isolate that
+		 * to be dealt with later.  The left-over data in
+		 * ccm_remainder_len from last time will not be part of the
+		 * MAC.  Otherwise, it would have already been taken out
+		 * when this call is made last time.
+		 */
+		size_t pt_part = pt_len - pd_len - ctx->ccm_remainder_len;
+
+		mac_len = length - pt_part;
+
+		ctx->ccm_processed_mac_len = mac_len;
+		bcopy(data + pt_part, ctx->ccm_mac_input_buf, mac_len);
+
+		if (pt_part + ctx->ccm_remainder_len < block_size) {
+			/*
+			 * since this is last of the ciphertext, will
+			 * just decrypt with it here
+			 */
+			bcopy(datap, &((uint8_t *)ctx->ccm_remainder)
+			    [ctx->ccm_remainder_len], pt_part);
+			ctx->ccm_remainder_len += pt_part;
+			ccm_decrypt_incomplete_block(ctx, encrypt_block);
+			ctx->ccm_processed_data_len += ctx->ccm_remainder_len;
+			ctx->ccm_remainder_len = 0;
+			return (CRYPTO_SUCCESS);
+		} else {
+			/* let rest of the code handle this */
+			length = pt_part;
+		}
+	} else if (length + ctx->ccm_remainder_len < block_size) {
+			/* accumulate bytes here and return */
+		bcopy(datap,
+		    (uint8_t *)ctx->ccm_remainder + ctx->ccm_remainder_len,
+		    length);
+		ctx->ccm_remainder_len += length;
+		ctx->ccm_copy_to = datap;
+		return (CRYPTO_SUCCESS);
+	}
+
+	do {
+		/* Unprocessed data from last call. */
+		if (ctx->ccm_remainder_len > 0) {
+			need = block_size - ctx->ccm_remainder_len;
+
+			if (need > remainder)
+				return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
+
+			bcopy(datap, &((uint8_t *)ctx->ccm_remainder)
+			    [ctx->ccm_remainder_len], need);
+
+			blockp = (uint8_t *)ctx->ccm_remainder;
+		} else {
+			blockp = datap;
+		}
+
+		/* Calculate the counter mode, ccm_cb is the counter block */
+		cbp = (uint8_t *)ctx->ccm_tmp;
+		encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, cbp);
+
+		/*
+		 * Increment counter.
+		 * Counter bits are confined to the bottom 64 bits
+		 */
+#ifdef _LITTLE_ENDIAN
+		counter = ntohll(ctx->ccm_cb[1] & ctx->ccm_counter_mask);
+		counter = htonll(counter + 1);
+#else
+		counter = ctx->ccm_cb[1] & ctx->ccm_counter_mask;
+		counter++;
+#endif	/* _LITTLE_ENDIAN */
+		counter &= ctx->ccm_counter_mask;
+		ctx->ccm_cb[1] =
+		    (ctx->ccm_cb[1] & ~(ctx->ccm_counter_mask)) | counter;
+
+		/* XOR with the ciphertext */
+		xor_block(blockp, cbp);
+
+		/* Copy the plaintext to the "holding buffer" */
+		resultp = (uint8_t *)ctx->ccm_pt_buf +
+		    ctx->ccm_processed_data_len;
+		copy_block(cbp, resultp);
+
+		ctx->ccm_processed_data_len += block_size;
+
+		ctx->ccm_lastp = blockp;
+
+		/* Update pointer to next block of data to be processed. */
+		if (ctx->ccm_remainder_len != 0) {
+			datap += need;
+			ctx->ccm_remainder_len = 0;
+		} else {
+			datap += block_size;
+		}
+
+		remainder = (size_t)&data[length] - (size_t)datap;
+
+		/* Incomplete last block */
+		if (remainder > 0 && remainder < block_size) {
+			bcopy(datap, ctx->ccm_remainder, remainder);
+			ctx->ccm_remainder_len = remainder;
+			ctx->ccm_copy_to = datap;
+			if (ctx->ccm_processed_mac_len > 0) {
+				/*
+				 * not expecting anymore ciphertext, just
+				 * compute plaintext for the remaining input
+				 */
+				ccm_decrypt_incomplete_block(ctx,
+				    encrypt_block);
+				ctx->ccm_processed_data_len += remainder;
+				ctx->ccm_remainder_len = 0;
+			}
+			goto out;
+		}
+		ctx->ccm_copy_to = NULL;
+
+	} while (remainder > 0);
+
+out:
+	return (CRYPTO_SUCCESS);
+}
+
+int
+ccm_decrypt_final(ccm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	size_t mac_remain, pt_len;
+	uint8_t *pt, *mac_buf, *macp, *ccm_mac_p;
+	int rv;
+
+	pt_len = ctx->ccm_data_len;
+
+	/* Make sure output buffer can fit all of the plaintext */
+	if (out->cd_length < pt_len) {
+		return (CRYPTO_DATA_LEN_RANGE);
+	}
+
+	pt = ctx->ccm_pt_buf;
+	mac_remain = ctx->ccm_processed_data_len;
+	mac_buf = (uint8_t *)ctx->ccm_mac_buf;
+
+	macp = (uint8_t *)ctx->ccm_tmp;
+
+	while (mac_remain > 0) {
+
+		if (mac_remain < block_size) {
+			bzero(macp, block_size);
+			bcopy(pt, macp, mac_remain);
+			mac_remain = 0;
+		} else {
+			copy_block(pt, macp);
+			mac_remain -= block_size;
+			pt += block_size;
+		}
+
+		/* calculate the CBC MAC */
+		xor_block(macp, mac_buf);
+		encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
+	}
+
+	/* Calculate the CCM MAC */
+	ccm_mac_p = (uint8_t *)ctx->ccm_tmp;
+	calculate_ccm_mac((ccm_ctx_t *)ctx, ccm_mac_p, encrypt_block);
+
+	/* compare the input CCM MAC value with what we calculated */
+	if (bcmp(ctx->ccm_mac_input_buf, ccm_mac_p, ctx->ccm_mac_len)) {
+		/* They don't match */
+		return (CRYPTO_INVALID_MAC);
+	} else {
+		rv = crypto_put_output_data(ctx->ccm_pt_buf, out, pt_len);
+		if (rv != CRYPTO_SUCCESS)
+			return (rv);
+		out->cd_offset += pt_len;
+	}
+	return (CRYPTO_SUCCESS);
+}
+
+int
+ccm_validate_args(CK_AES_CCM_PARAMS *ccm_param, boolean_t is_encrypt_init)
+{
+	size_t macSize, nonceSize;
+	uint8_t q;
+	uint64_t maxValue;
+
+	/*
+	 * Check the length of the MAC.  The only valid
+	 * lengths for the MAC are: 4, 6, 8, 10, 12, 14, 16
+	 */
+	macSize = ccm_param->ulMACSize;
+	if ((macSize < 4) || (macSize > 16) || ((macSize % 2) != 0)) {
+		return (CRYPTO_MECHANISM_PARAM_INVALID);
+	}
+
+	/* Check the nonce length.  Valid values are 7, 8, 9, 10, 11, 12, 13 */
+	nonceSize = ccm_param->ulNonceSize;
+	if ((nonceSize < 7) || (nonceSize > 13)) {
+		return (CRYPTO_MECHANISM_PARAM_INVALID);
+	}
+
+	/* q is the length of the field storing the length, in bytes */
+	q = (uint8_t)((15 - nonceSize) & 0xFF);
+
+
+	/*
+	 * If it is decrypt, need to make sure size of ciphertext is at least
+	 * bigger than MAC len
+	 */
+	if ((!is_encrypt_init) && (ccm_param->ulDataSize < macSize)) {
+		return (CRYPTO_MECHANISM_PARAM_INVALID);
+	}
+
+	/*
+	 * Check to make sure the length of the payload is within the
+	 * range of values allowed by q
+	 */
+	if (q < 8) {
+		maxValue = (1ULL << (q * 8)) - 1;
+	} else {
+		maxValue = ULONG_MAX;
+	}
+
+	if (ccm_param->ulDataSize > maxValue) {
+		return (CRYPTO_MECHANISM_PARAM_INVALID);
+	}
+	return (CRYPTO_SUCCESS);
+}
+
+/*
+ * Format the first block used in CBC-MAC (B0) and the initial counter
+ * block based on formatting functions and counter generation functions
+ * specified in RFC 3610 and NIST publication 800-38C, appendix A
+ *
+ * b0 is the first block used in CBC-MAC
+ * cb0 is the first counter block
+ *
+ * It's assumed that the arguments b0 and cb0 are preallocated AES blocks
+ *
+ */
+static void
+ccm_format_initial_blocks(uchar_t *nonce, ulong_t nonceSize,
+    ulong_t authDataSize, uint8_t *b0, ccm_ctx_t *aes_ctx)
+{
+	uint64_t payloadSize;
+	uint8_t t, q, have_adata = 0;
+	size_t limit;
+	int i, j, k;
+	uint64_t mask = 0;
+	uint8_t *cb;
+
+	q = (uint8_t)((15 - nonceSize) & 0xFF);
+	t = (uint8_t)((aes_ctx->ccm_mac_len) & 0xFF);
+
+	/* Construct the first octet of b0 */
+	if (authDataSize > 0) {
+		have_adata = 1;
+	}
+	b0[0] = (have_adata << 6) | (((t - 2)  / 2) << 3) | (q - 1);
+
+	/* copy the nonce value into b0 */
+	bcopy(nonce, &(b0[1]), nonceSize);
+
+	/* store the length of the payload into b0 */
+	bzero(&(b0[1+nonceSize]), q);
+
+	payloadSize = aes_ctx->ccm_data_len;
+	limit = 8 < q ? 8 : q;
+
+	for (i = 0, j = 0, k = 15; i < limit; i++, j += 8, k--) {
+		b0[k] = (uint8_t)((payloadSize >> j) & 0xFF);
+	}
+
+	/* format the counter block */
+
+	cb = (uint8_t *)aes_ctx->ccm_cb;
+
+	cb[0] = 0x07 & (q-1); /* first byte */
+
+	/* copy the nonce value into the counter block */
+	bcopy(nonce, &(cb[1]), nonceSize);
+
+	bzero(&(cb[1+nonceSize]), q);
+
+	/* Create the mask for the counter field based on the size of nonce */
+	q <<= 3;
+	while (q-- > 0) {
+		mask |= (1ULL << q);
+	}
+
+#ifdef _LITTLE_ENDIAN
+	mask = htonll(mask);
+#endif
+	aes_ctx->ccm_counter_mask = mask;
+
+	/*
+	 * During calculation, we start using counter block 1, we will
+	 * set it up right here.
+	 * We can just set the last byte to have the value 1, because
+	 * even with the biggest nonce of 13, the last byte of the
+	 * counter block will be used for the counter value.
+	 */
+	cb[15] = 0x01;
+}
+
+/*
+ * Encode the length of the associated data as
+ * specified in RFC 3610 and NIST publication 800-38C, appendix A
+ */
+static void
+encode_adata_len(ulong_t auth_data_len, uint8_t *encoded, size_t *encoded_len)
+{
+#ifdef UNALIGNED_POINTERS_PERMITTED
+	uint32_t	*lencoded_ptr;
+#ifdef _LP64
+	uint64_t	*llencoded_ptr;
+#endif
+#endif	/* UNALIGNED_POINTERS_PERMITTED */
+
+	if (auth_data_len < ((1ULL<<16) - (1ULL<<8))) {
+		/* 0 < a < (2^16-2^8) */
+		*encoded_len = 2;
+		encoded[0] = (auth_data_len & 0xff00) >> 8;
+		encoded[1] = auth_data_len & 0xff;
+
+	} else if ((auth_data_len >= ((1ULL<<16) - (1ULL<<8))) &&
+	    (auth_data_len < (1ULL << 31))) {
+		/* (2^16-2^8) <= a < 2^32 */
+		*encoded_len = 6;
+		encoded[0] = 0xff;
+		encoded[1] = 0xfe;
+#ifdef UNALIGNED_POINTERS_PERMITTED
+		lencoded_ptr = (uint32_t *)&encoded[2];
+		*lencoded_ptr = htonl(auth_data_len);
+#else
+		encoded[2] = (auth_data_len & 0xff000000) >> 24;
+		encoded[3] = (auth_data_len & 0xff0000) >> 16;
+		encoded[4] = (auth_data_len & 0xff00) >> 8;
+		encoded[5] = auth_data_len & 0xff;
+#endif	/* UNALIGNED_POINTERS_PERMITTED */
+
+#ifdef _LP64
+	} else {
+		/* 2^32 <= a < 2^64 */
+		*encoded_len = 10;
+		encoded[0] = 0xff;
+		encoded[1] = 0xff;
+#ifdef UNALIGNED_POINTERS_PERMITTED
+		llencoded_ptr = (uint64_t *)&encoded[2];
+		*llencoded_ptr = htonl(auth_data_len);
+#else
+		encoded[2] = (auth_data_len & 0xff00000000000000) >> 56;
+		encoded[3] = (auth_data_len & 0xff000000000000) >> 48;
+		encoded[4] = (auth_data_len & 0xff0000000000) >> 40;
+		encoded[5] = (auth_data_len & 0xff00000000) >> 32;
+		encoded[6] = (auth_data_len & 0xff000000) >> 24;
+		encoded[7] = (auth_data_len & 0xff0000) >> 16;
+		encoded[8] = (auth_data_len & 0xff00) >> 8;
+		encoded[9] = auth_data_len & 0xff;
+#endif	/* UNALIGNED_POINTERS_PERMITTED */
+#endif	/* _LP64 */
+	}
+}
+
+/*
+ * The following function should be call at encrypt or decrypt init time
+ * for AES CCM mode.
+ */
+int
+ccm_init(ccm_ctx_t *ctx, unsigned char *nonce, size_t nonce_len,
+    unsigned char *auth_data, size_t auth_data_len, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	uint8_t *mac_buf, *datap, *ivp, *authp;
+	size_t remainder, processed;
+	uint8_t encoded_a[10]; /* max encoded auth data length is 10 octets */
+	size_t encoded_a_len = 0;
+
+	mac_buf = (uint8_t *)&(ctx->ccm_mac_buf);
+
+	/*
+	 * Format the 1st block for CBC-MAC and construct the
+	 * 1st counter block.
+	 *
+	 * aes_ctx->ccm_iv is used for storing the counter block
+	 * mac_buf will store b0 at this time.
+	 */
+	ccm_format_initial_blocks(nonce, nonce_len,
+	    auth_data_len, mac_buf, ctx);
+
+	/* The IV for CBC MAC for AES CCM mode is always zero */
+	ivp = (uint8_t *)ctx->ccm_tmp;
+	bzero(ivp, block_size);
+
+	xor_block(ivp, mac_buf);
+
+	/* encrypt the nonce */
+	encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
+
+	/* take care of the associated data, if any */
+	if (auth_data_len == 0) {
+		return (CRYPTO_SUCCESS);
+	}
+
+	encode_adata_len(auth_data_len, encoded_a, &encoded_a_len);
+
+	remainder = auth_data_len;
+
+	/* 1st block: it contains encoded associated data, and some data */
+	authp = (uint8_t *)ctx->ccm_tmp;
+	bzero(authp, block_size);
+	bcopy(encoded_a, authp, encoded_a_len);
+	processed = block_size - encoded_a_len;
+	if (processed > auth_data_len) {
+		/* in case auth_data is very small */
+		processed = auth_data_len;
+	}
+	bcopy(auth_data, authp+encoded_a_len, processed);
+	/* xor with previous buffer */
+	xor_block(authp, mac_buf);
+	encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
+	remainder -= processed;
+	if (remainder == 0) {
+		/* a small amount of associated data, it's all done now */
+		return (CRYPTO_SUCCESS);
+	}
+
+	do {
+		if (remainder < block_size) {
+			/*
+			 * There's not a block full of data, pad rest of
+			 * buffer with zero
+			 */
+			bzero(authp, block_size);
+			bcopy(&(auth_data[processed]), authp, remainder);
+			datap = (uint8_t *)authp;
+			remainder = 0;
+		} else {
+			datap = (uint8_t *)(&(auth_data[processed]));
+			processed += block_size;
+			remainder -= block_size;
+		}
+
+		xor_block(datap, mac_buf);
+		encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
+
+	} while (remainder > 0);
+
+	return (CRYPTO_SUCCESS);
+}
+
+int
+ccm_init_ctx(ccm_ctx_t *ccm_ctx, char *param, int kmflag,
+    boolean_t is_encrypt_init, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	int rv;
+	CK_AES_CCM_PARAMS *ccm_param;
+
+	if (param != NULL) {
+		ccm_param = (CK_AES_CCM_PARAMS *)param;
+
+		if ((rv = ccm_validate_args(ccm_param,
+		    is_encrypt_init)) != 0) {
+			return (rv);
+		}
+
+		ccm_ctx->ccm_mac_len = ccm_param->ulMACSize;
+		if (is_encrypt_init) {
+			ccm_ctx->ccm_data_len = ccm_param->ulDataSize;
+		} else {
+			ccm_ctx->ccm_data_len =
+			    ccm_param->ulDataSize - ccm_ctx->ccm_mac_len;
+			ccm_ctx->ccm_processed_mac_len = 0;
+		}
+		ccm_ctx->ccm_processed_data_len = 0;
+
+		ccm_ctx->ccm_flags |= CCM_MODE;
+	} else {
+		rv = CRYPTO_MECHANISM_PARAM_INVALID;
+		goto out;
+	}
+
+	if (ccm_init(ccm_ctx, ccm_param->nonce, ccm_param->ulNonceSize,
+	    ccm_param->authData, ccm_param->ulAuthDataSize, block_size,
+	    encrypt_block, xor_block) != 0) {
+		rv = CRYPTO_MECHANISM_PARAM_INVALID;
+		goto out;
+	}
+	if (!is_encrypt_init) {
+		/* allocate buffer for storing decrypted plaintext */
+		ccm_ctx->ccm_pt_buf = vmem_alloc(ccm_ctx->ccm_data_len,
+		    kmflag);
+		if (ccm_ctx->ccm_pt_buf == NULL) {
+			rv = CRYPTO_HOST_MEMORY;
+		}
+	}
+out:
+	return (rv);
+}
+
+void *
+ccm_alloc_ctx(int kmflag)
+{
+	ccm_ctx_t *ccm_ctx;
+
+	if ((ccm_ctx = kmem_zalloc(sizeof (ccm_ctx_t), kmflag)) == NULL)
+		return (NULL);
+
+	ccm_ctx->ccm_flags = CCM_MODE;
+	return (ccm_ctx);
+}

module/icp/algs/modes/ctr.c

@@ -0,0 +1,238 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+#include <sys/byteorder.h>
+
+/*
+ * Encrypt and decrypt multiple blocks of data in counter mode.
+ */
+int
+ctr_mode_contiguous_blocks(ctr_ctx_t *ctx, char *data, size_t length,
+    crypto_data_t *out, size_t block_size,
+    int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	size_t remainder = length;
+	size_t need = 0;
+	uint8_t *datap = (uint8_t *)data;
+	uint8_t *blockp;
+	uint8_t *lastp;
+	void *iov_or_mp;
+	offset_t offset;
+	uint8_t *out_data_1;
+	uint8_t *out_data_2;
+	size_t out_data_1_len;
+	uint64_t lower_counter, upper_counter;
+
+	if (length + ctx->ctr_remainder_len < block_size) {
+		/* accumulate bytes here and return */
+		bcopy(datap,
+		    (uint8_t *)ctx->ctr_remainder + ctx->ctr_remainder_len,
+		    length);
+		ctx->ctr_remainder_len += length;
+		ctx->ctr_copy_to = datap;
+		return (CRYPTO_SUCCESS);
+	}
+
+	lastp = (uint8_t *)ctx->ctr_cb;
+	if (out != NULL)
+		crypto_init_ptrs(out, &iov_or_mp, &offset);
+
+	do {
+		/* Unprocessed data from last call. */
+		if (ctx->ctr_remainder_len > 0) {
+			need = block_size - ctx->ctr_remainder_len;
+
+			if (need > remainder)
+				return (CRYPTO_DATA_LEN_RANGE);
+
+			bcopy(datap, &((uint8_t *)ctx->ctr_remainder)
+			    [ctx->ctr_remainder_len], need);
+
+			blockp = (uint8_t *)ctx->ctr_remainder;
+		} else {
+			blockp = datap;
+		}
+
+		/* ctr_cb is the counter block */
+		cipher(ctx->ctr_keysched, (uint8_t *)ctx->ctr_cb,
+		    (uint8_t *)ctx->ctr_tmp);
+
+		lastp = (uint8_t *)ctx->ctr_tmp;
+
+		/*
+		 * Increment Counter.
+		 */
+		lower_counter = ntohll(ctx->ctr_cb[1] & ctx->ctr_lower_mask);
+		lower_counter = htonll(lower_counter + 1);
+		lower_counter &= ctx->ctr_lower_mask;
+		ctx->ctr_cb[1] = (ctx->ctr_cb[1] & ~(ctx->ctr_lower_mask)) |
+		    lower_counter;
+
+		/* wrap around */
+		if (lower_counter == 0) {
+			upper_counter =
+			    ntohll(ctx->ctr_cb[0] & ctx->ctr_upper_mask);
+			upper_counter = htonll(upper_counter + 1);
+			upper_counter &= ctx->ctr_upper_mask;
+			ctx->ctr_cb[0] =
+			    (ctx->ctr_cb[0] & ~(ctx->ctr_upper_mask)) |
+			    upper_counter;
+		}
+
+		/*
+		 * XOR encrypted counter block with the current clear block.
+		 */
+		xor_block(blockp, lastp);
+
+		if (out == NULL) {
+			if (ctx->ctr_remainder_len > 0) {
+				bcopy(lastp, ctx->ctr_copy_to,
+				    ctx->ctr_remainder_len);
+				bcopy(lastp + ctx->ctr_remainder_len, datap,
+				    need);
+			}
+		} else {
+			crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+			    &out_data_1_len, &out_data_2, block_size);
+
+			/* copy block to where it belongs */
+			bcopy(lastp, out_data_1, out_data_1_len);
+			if (out_data_2 != NULL) {
+				bcopy(lastp + out_data_1_len, out_data_2,
+				    block_size - out_data_1_len);
+			}
+			/* update offset */
+			out->cd_offset += block_size;
+		}
+
+		/* Update pointer to next block of data to be processed. */
+		if (ctx->ctr_remainder_len != 0) {
+			datap += need;
+			ctx->ctr_remainder_len = 0;
+		} else {
+			datap += block_size;
+		}
+
+		remainder = (size_t)&data[length] - (size_t)datap;
+
+		/* Incomplete last block. */
+		if (remainder > 0 && remainder < block_size) {
+			bcopy(datap, ctx->ctr_remainder, remainder);
+			ctx->ctr_remainder_len = remainder;
+			ctx->ctr_copy_to = datap;
+			goto out;
+		}
+		ctx->ctr_copy_to = NULL;
+
+	} while (remainder > 0);
+
+out:
+	return (CRYPTO_SUCCESS);
+}
+
+int
+ctr_mode_final(ctr_ctx_t *ctx, crypto_data_t *out,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *))
+{
+	uint8_t *lastp;
+	void *iov_or_mp;
+	offset_t offset;
+	uint8_t *out_data_1;
+	uint8_t *out_data_2;
+	size_t out_data_1_len;
+	uint8_t *p;
+	int i;
+
+	if (out->cd_length < ctx->ctr_remainder_len)
+		return (CRYPTO_DATA_LEN_RANGE);
+
+	encrypt_block(ctx->ctr_keysched, (uint8_t *)ctx->ctr_cb,
+	    (uint8_t *)ctx->ctr_tmp);
+
+	lastp = (uint8_t *)ctx->ctr_tmp;
+	p = (uint8_t *)ctx->ctr_remainder;
+	for (i = 0; i < ctx->ctr_remainder_len; i++) {
+		p[i] ^= lastp[i];
+	}
+
+	crypto_init_ptrs(out, &iov_or_mp, &offset);
+	crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+	    &out_data_1_len, &out_data_2, ctx->ctr_remainder_len);
+
+	bcopy(p, out_data_1, out_data_1_len);
+	if (out_data_2 != NULL) {
+		bcopy((uint8_t *)p + out_data_1_len,
+		    out_data_2, ctx->ctr_remainder_len - out_data_1_len);
+	}
+	out->cd_offset += ctx->ctr_remainder_len;
+	ctx->ctr_remainder_len = 0;
+	return (CRYPTO_SUCCESS);
+}
+
+int
+ctr_init_ctx(ctr_ctx_t *ctr_ctx, ulong_t count, uint8_t *cb,
+void (*copy_block)(uint8_t *, uint8_t *))
+{
+	uint64_t upper_mask = 0;
+	uint64_t lower_mask = 0;
+
+	if (count == 0 || count > 128) {
+		return (CRYPTO_MECHANISM_PARAM_INVALID);
+	}
+	/* upper 64 bits of the mask */
+	if (count >= 64) {
+		count -= 64;
+		upper_mask = (count == 64) ? UINT64_MAX : (1ULL << count) - 1;
+		lower_mask = UINT64_MAX;
+	} else {
+		/* now the lower 63 bits */
+		lower_mask = (1ULL << count) - 1;
+	}
+	ctr_ctx->ctr_lower_mask = htonll(lower_mask);
+	ctr_ctx->ctr_upper_mask = htonll(upper_mask);
+
+	copy_block(cb, (uchar_t *)ctr_ctx->ctr_cb);
+	ctr_ctx->ctr_lastp = (uint8_t *)&ctr_ctx->ctr_cb[0];
+	ctr_ctx->ctr_flags |= CTR_MODE;
+	return (CRYPTO_SUCCESS);
+}
+
+/* ARGSUSED */
+void *
+ctr_alloc_ctx(int kmflag)
+{
+	ctr_ctx_t *ctr_ctx;
+
+	if ((ctr_ctx = kmem_zalloc(sizeof (ctr_ctx_t), kmflag)) == NULL)
+		return (NULL);
+
+	ctr_ctx->ctr_flags = CTR_MODE;
+	return (ctr_ctx);
+}

module/icp/algs/modes/ecb.c

@@ -0,0 +1,143 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+
+/*
+ * Algorithm independent ECB functions.
+ */
+int
+ecb_cipher_contiguous_blocks(ecb_ctx_t *ctx, char *data, size_t length,
+    crypto_data_t *out, size_t block_size,
+    int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct))
+{
+	size_t remainder = length;
+	size_t need = 0;
+	uint8_t *datap = (uint8_t *)data;
+	uint8_t *blockp;
+	uint8_t *lastp;
+	void *iov_or_mp;
+	offset_t offset;
+	uint8_t *out_data_1;
+	uint8_t *out_data_2;
+	size_t out_data_1_len;
+
+	if (length + ctx->ecb_remainder_len < block_size) {
+		/* accumulate bytes here and return */
+		bcopy(datap,
+		    (uint8_t *)ctx->ecb_remainder + ctx->ecb_remainder_len,
+		    length);
+		ctx->ecb_remainder_len += length;
+		ctx->ecb_copy_to = datap;
+		return (CRYPTO_SUCCESS);
+	}
+
+	lastp = (uint8_t *)ctx->ecb_iv;
+	if (out != NULL)
+		crypto_init_ptrs(out, &iov_or_mp, &offset);
+
+	do {
+		/* Unprocessed data from last call. */
+		if (ctx->ecb_remainder_len > 0) {
+			need = block_size - ctx->ecb_remainder_len;
+
+			if (need > remainder)
+				return (CRYPTO_DATA_LEN_RANGE);
+
+			bcopy(datap, &((uint8_t *)ctx->ecb_remainder)
+			    [ctx->ecb_remainder_len], need);
+
+			blockp = (uint8_t *)ctx->ecb_remainder;
+		} else {
+			blockp = datap;
+		}
+
+		if (out == NULL) {
+			cipher(ctx->ecb_keysched, blockp, blockp);
+
+			ctx->ecb_lastp = blockp;
+			lastp = blockp;
+
+			if (ctx->ecb_remainder_len > 0) {
+				bcopy(blockp, ctx->ecb_copy_to,
+				    ctx->ecb_remainder_len);
+				bcopy(blockp + ctx->ecb_remainder_len, datap,
+				    need);
+			}
+		} else {
+			cipher(ctx->ecb_keysched, blockp, lastp);
+			crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+			    &out_data_1_len, &out_data_2, block_size);
+
+			/* copy block to where it belongs */
+			bcopy(lastp, out_data_1, out_data_1_len);
+			if (out_data_2 != NULL) {
+				bcopy(lastp + out_data_1_len, out_data_2,
+				    block_size - out_data_1_len);
+			}
+			/* update offset */
+			out->cd_offset += block_size;
+		}
+
+		/* Update pointer to next block of data to be processed. */
+		if (ctx->ecb_remainder_len != 0) {
+			datap += need;
+			ctx->ecb_remainder_len = 0;
+		} else {
+			datap += block_size;
+		}
+
+		remainder = (size_t)&data[length] - (size_t)datap;
+
+		/* Incomplete last block. */
+		if (remainder > 0 && remainder < block_size) {
+			bcopy(datap, ctx->ecb_remainder, remainder);
+			ctx->ecb_remainder_len = remainder;
+			ctx->ecb_copy_to = datap;
+			goto out;
+		}
+		ctx->ecb_copy_to = NULL;
+
+	} while (remainder > 0);
+
+out:
+	return (CRYPTO_SUCCESS);
+}
+
+/* ARGSUSED */
+void *
+ecb_alloc_ctx(int kmflag)
+{
+	ecb_ctx_t *ecb_ctx;
+
+	if ((ecb_ctx = kmem_zalloc(sizeof (ecb_ctx_t), kmflag)) == NULL)
+		return (NULL);
+
+	ecb_ctx->ecb_flags = ECB_MODE;
+	return (ecb_ctx);
+}

module/icp/algs/modes/gcm.c

@@ -0,0 +1,748 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+#include <sys/byteorder.h>
+
+#ifdef __amd64
+
+#ifdef _KERNEL
+/* Workaround for no XMM kernel thread save/restore */
+#define	KPREEMPT_DISABLE	kpreempt_disable()
+#define	KPREEMPT_ENABLE		kpreempt_enable()
+
+#else
+#define	KPREEMPT_DISABLE
+#define	KPREEMPT_ENABLE
+#endif	/* _KERNEL */
+
+extern void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res);
+static int intel_pclmulqdq_instruction_present(void);
+#endif	/* __amd64 */
+
+struct aes_block {
+	uint64_t a;
+	uint64_t b;
+};
+
+
+/*
+ * gcm_mul()
+ * Perform a carry-less multiplication (that is, use XOR instead of the
+ * multiply operator) on *x_in and *y and place the result in *res.
+ *
+ * Byte swap the input (*x_in and *y) and the output (*res).
+ *
+ * Note: x_in, y, and res all point to 16-byte numbers (an array of two
+ * 64-bit integers).
+ */
+void
+gcm_mul(uint64_t *x_in, uint64_t *y, uint64_t *res)
+{
+#ifdef __amd64
+	if (intel_pclmulqdq_instruction_present()) {
+		KPREEMPT_DISABLE;
+		gcm_mul_pclmulqdq(x_in, y, res);
+		KPREEMPT_ENABLE;
+	} else
+#endif	/* __amd64 */
+	{
+		static const uint64_t R = 0xe100000000000000ULL;
+		struct aes_block z = {0, 0};
+		struct aes_block v;
+		uint64_t x;
+		int i, j;
+
+		v.a = ntohll(y[0]);
+		v.b = ntohll(y[1]);
+
+		for (j = 0; j < 2; j++) {
+			x = ntohll(x_in[j]);
+			for (i = 0; i < 64; i++, x <<= 1) {
+				if (x & 0x8000000000000000ULL) {
+					z.a ^= v.a;
+					z.b ^= v.b;
+				}
+				if (v.b & 1ULL) {
+					v.b = (v.a << 63)|(v.b >> 1);
+					v.a = (v.a >> 1) ^ R;
+				} else {
+					v.b = (v.a << 63)|(v.b >> 1);
+					v.a = v.a >> 1;
+				}
+			}
+		}
+		res[0] = htonll(z.a);
+		res[1] = htonll(z.b);
+	}
+}
+
+
+#define	GHASH(c, d, t) \
+	xor_block((uint8_t *)(d), (uint8_t *)(c)->gcm_ghash); \
+	gcm_mul((uint64_t *)(void *)(c)->gcm_ghash, (c)->gcm_H, \
+	(uint64_t *)(void *)(t));
+
+
+/*
+ * Encrypt multiple blocks of data in GCM mode.  Decrypt for GCM mode
+ * is done in another function.
+ */
+int
+gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
+    crypto_data_t *out, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	size_t remainder = length;
+	size_t need = 0;
+	uint8_t *datap = (uint8_t *)data;
+	uint8_t *blockp;
+	uint8_t *lastp;
+	void *iov_or_mp;
+	offset_t offset;
+	uint8_t *out_data_1;
+	uint8_t *out_data_2;
+	size_t out_data_1_len;
+	uint64_t counter;
+	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
+
+	if (length + ctx->gcm_remainder_len < block_size) {
+		/* accumulate bytes here and return */
+		bcopy(datap,
+		    (uint8_t *)ctx->gcm_remainder + ctx->gcm_remainder_len,
+		    length);
+		ctx->gcm_remainder_len += length;
+		ctx->gcm_copy_to = datap;
+		return (CRYPTO_SUCCESS);
+	}
+
+	lastp = (uint8_t *)ctx->gcm_cb;
+	if (out != NULL)
+		crypto_init_ptrs(out, &iov_or_mp, &offset);
+
+	do {
+		/* Unprocessed data from last call. */
+		if (ctx->gcm_remainder_len > 0) {
+			need = block_size - ctx->gcm_remainder_len;
+
+			if (need > remainder)
+				return (CRYPTO_DATA_LEN_RANGE);
+
+			bcopy(datap, &((uint8_t *)ctx->gcm_remainder)
+			    [ctx->gcm_remainder_len], need);
+
+			blockp = (uint8_t *)ctx->gcm_remainder;
+		} else {
+			blockp = datap;
+		}
+
+		/*
+		 * Increment counter. Counter bits are confined
+		 * to the bottom 32 bits of the counter block.
+		 */
+		counter = ntohll(ctx->gcm_cb[1] & counter_mask);
+		counter = htonll(counter + 1);
+		counter &= counter_mask;
+		ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
+
+		encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb,
+		    (uint8_t *)ctx->gcm_tmp);
+		xor_block(blockp, (uint8_t *)ctx->gcm_tmp);
+
+		lastp = (uint8_t *)ctx->gcm_tmp;
+
+		ctx->gcm_processed_data_len += block_size;
+
+		if (out == NULL) {
+			if (ctx->gcm_remainder_len > 0) {
+				bcopy(blockp, ctx->gcm_copy_to,
+				    ctx->gcm_remainder_len);
+				bcopy(blockp + ctx->gcm_remainder_len, datap,
+				    need);
+			}
+		} else {
+			crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
+			    &out_data_1_len, &out_data_2, block_size);
+
+			/* copy block to where it belongs */
+			if (out_data_1_len == block_size) {
+				copy_block(lastp, out_data_1);
+			} else {
+				bcopy(lastp, out_data_1, out_data_1_len);
+				if (out_data_2 != NULL) {
+					bcopy(lastp + out_data_1_len,
+					    out_data_2,
+					    block_size - out_data_1_len);
+				}
+			}
+			/* update offset */
+			out->cd_offset += block_size;
+		}
+
+		/* add ciphertext to the hash */
+		GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
+
+		/* Update pointer to next block of data to be processed. */
+		if (ctx->gcm_remainder_len != 0) {
+			datap += need;
+			ctx->gcm_remainder_len = 0;
+		} else {
+			datap += block_size;
+		}
+
+		remainder = (size_t)&data[length] - (size_t)datap;
+
+		/* Incomplete last block. */
+		if (remainder > 0 && remainder < block_size) {
+			bcopy(datap, ctx->gcm_remainder, remainder);
+			ctx->gcm_remainder_len = remainder;
+			ctx->gcm_copy_to = datap;
+			goto out;
+		}
+		ctx->gcm_copy_to = NULL;
+
+	} while (remainder > 0);
+out:
+	return (CRYPTO_SUCCESS);
+}
+
+/* ARGSUSED */
+int
+gcm_encrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
+	uint8_t *ghash, *macp = NULL;
+	int i, rv;
+
+	if (out->cd_length <
+	    (ctx->gcm_remainder_len + ctx->gcm_tag_len)) {
+		return (CRYPTO_DATA_LEN_RANGE);
+	}
+
+	ghash = (uint8_t *)ctx->gcm_ghash;
+
+	if (ctx->gcm_remainder_len > 0) {
+		uint64_t counter;
+		uint8_t *tmpp = (uint8_t *)ctx->gcm_tmp;
+
+		/*
+		 * Here is where we deal with data that is not a
+		 * multiple of the block size.
+		 */
+
+		/*
+		 * Increment counter.
+		 */
+		counter = ntohll(ctx->gcm_cb[1] & counter_mask);
+		counter = htonll(counter + 1);
+		counter &= counter_mask;
+		ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
+
+		encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb,
+		    (uint8_t *)ctx->gcm_tmp);
+
+		macp = (uint8_t *)ctx->gcm_remainder;
+		bzero(macp + ctx->gcm_remainder_len,
+		    block_size - ctx->gcm_remainder_len);
+
+		/* XOR with counter block */
+		for (i = 0; i < ctx->gcm_remainder_len; i++) {
+			macp[i] ^= tmpp[i];
+		}
+
+		/* add ciphertext to the hash */
+		GHASH(ctx, macp, ghash);
+
+		ctx->gcm_processed_data_len += ctx->gcm_remainder_len;
+	}
+
+	ctx->gcm_len_a_len_c[1] =
+	    htonll(CRYPTO_BYTES2BITS(ctx->gcm_processed_data_len));
+	GHASH(ctx, ctx->gcm_len_a_len_c, ghash);
+	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0,
+	    (uint8_t *)ctx->gcm_J0);
+	xor_block((uint8_t *)ctx->gcm_J0, ghash);
+
+	if (ctx->gcm_remainder_len > 0) {
+		rv = crypto_put_output_data(macp, out, ctx->gcm_remainder_len);
+		if (rv != CRYPTO_SUCCESS)
+			return (rv);
+	}
+	out->cd_offset += ctx->gcm_remainder_len;
+	ctx->gcm_remainder_len = 0;
+	rv = crypto_put_output_data(ghash, out, ctx->gcm_tag_len);
+	if (rv != CRYPTO_SUCCESS)
+		return (rv);
+	out->cd_offset += ctx->gcm_tag_len;
+
+	return (CRYPTO_SUCCESS);
+}
+
+/*
+ * This will only deal with decrypting the last block of the input that
+ * might not be a multiple of block length.
+ */
+static void
+gcm_decrypt_incomplete_block(gcm_ctx_t *ctx, size_t block_size, size_t index,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	uint8_t *datap, *outp, *counterp;
+	uint64_t counter;
+	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
+	int i;
+
+	/*
+	 * Increment counter.
+	 * Counter bits are confined to the bottom 32 bits
+	 */
+	counter = ntohll(ctx->gcm_cb[1] & counter_mask);
+	counter = htonll(counter + 1);
+	counter &= counter_mask;
+	ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
+
+	datap = (uint8_t *)ctx->gcm_remainder;
+	outp = &((ctx->gcm_pt_buf)[index]);
+	counterp = (uint8_t *)ctx->gcm_tmp;
+
+	/* authentication tag */
+	bzero((uint8_t *)ctx->gcm_tmp, block_size);
+	bcopy(datap, (uint8_t *)ctx->gcm_tmp, ctx->gcm_remainder_len);
+
+	/* add ciphertext to the hash */
+	GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
+
+	/* decrypt remaining ciphertext */
+	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, counterp);
+
+	/* XOR with counter block */
+	for (i = 0; i < ctx->gcm_remainder_len; i++) {
+		outp[i] = datap[i] ^ counterp[i];
+	}
+}
+
+/* ARGSUSED */
+int
+gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
+    crypto_data_t *out, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	size_t new_len;
+	uint8_t *new;
+
+	/*
+	 * Copy contiguous ciphertext input blocks to plaintext buffer.
+	 * Ciphertext will be decrypted in the final.
+	 */
+	if (length > 0) {
+		new_len = ctx->gcm_pt_buf_len + length;
+		new = vmem_alloc(new_len, ctx->gcm_kmflag);
+		bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len);
+		vmem_free(ctx->gcm_pt_buf, ctx->gcm_pt_buf_len);
+		if (new == NULL)
+			return (CRYPTO_HOST_MEMORY);
+
+		ctx->gcm_pt_buf = new;
+		ctx->gcm_pt_buf_len = new_len;
+		bcopy(data, &ctx->gcm_pt_buf[ctx->gcm_processed_data_len],
+		    length);
+		ctx->gcm_processed_data_len += length;
+	}
+
+	ctx->gcm_remainder_len = 0;
+	return (CRYPTO_SUCCESS);
+}
+
+int
+gcm_decrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	size_t pt_len;
+	size_t remainder;
+	uint8_t *ghash;
+	uint8_t *blockp;
+	uint8_t *cbp;
+	uint64_t counter;
+	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
+	int processed = 0, rv;
+
+	ASSERT(ctx->gcm_processed_data_len == ctx->gcm_pt_buf_len);
+
+	pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
+	ghash = (uint8_t *)ctx->gcm_ghash;
+	blockp = ctx->gcm_pt_buf;
+	remainder = pt_len;
+	while (remainder > 0) {
+		/* Incomplete last block */
+		if (remainder < block_size) {
+			bcopy(blockp, ctx->gcm_remainder, remainder);
+			ctx->gcm_remainder_len = remainder;
+			/*
+			 * not expecting anymore ciphertext, just
+			 * compute plaintext for the remaining input
+			 */
+			gcm_decrypt_incomplete_block(ctx, block_size,
+			    processed, encrypt_block, xor_block);
+			ctx->gcm_remainder_len = 0;
+			goto out;
+		}
+		/* add ciphertext to the hash */
+		GHASH(ctx, blockp, ghash);
+
+		/*
+		 * Increment counter.
+		 * Counter bits are confined to the bottom 32 bits
+		 */
+		counter = ntohll(ctx->gcm_cb[1] & counter_mask);
+		counter = htonll(counter + 1);
+		counter &= counter_mask;
+		ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
+
+		cbp = (uint8_t *)ctx->gcm_tmp;
+		encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, cbp);
+
+		/* XOR with ciphertext */
+		xor_block(cbp, blockp);
+
+		processed += block_size;
+		blockp += block_size;
+		remainder -= block_size;
+	}
+out:
+	ctx->gcm_len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(pt_len));
+	GHASH(ctx, ctx->gcm_len_a_len_c, ghash);
+	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0,
+	    (uint8_t *)ctx->gcm_J0);
+	xor_block((uint8_t *)ctx->gcm_J0, ghash);
+
+	/* compare the input authentication tag with what we calculated */
+	if (bcmp(&ctx->gcm_pt_buf[pt_len], ghash, ctx->gcm_tag_len)) {
+		/* They don't match */
+		return (CRYPTO_INVALID_MAC);
+	} else {
+		rv = crypto_put_output_data(ctx->gcm_pt_buf, out, pt_len);
+		if (rv != CRYPTO_SUCCESS)
+			return (rv);
+		out->cd_offset += pt_len;
+	}
+	return (CRYPTO_SUCCESS);
+}
+
+static int
+gcm_validate_args(CK_AES_GCM_PARAMS *gcm_param)
+{
+	size_t tag_len;
+
+	/*
+	 * Check the length of the authentication tag (in bits).
+	 */
+	tag_len = gcm_param->ulTagBits;
+	switch (tag_len) {
+	case 32:
+	case 64:
+	case 96:
+	case 104:
+	case 112:
+	case 120:
+	case 128:
+		break;
+	default:
+		return (CRYPTO_MECHANISM_PARAM_INVALID);
+	}
+
+	if (gcm_param->ulIvLen == 0)
+		return (CRYPTO_MECHANISM_PARAM_INVALID);
+
+	return (CRYPTO_SUCCESS);
+}
+
+static void
+gcm_format_initial_blocks(uchar_t *iv, ulong_t iv_len,
+    gcm_ctx_t *ctx, size_t block_size,
+    void (*copy_block)(uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	uint8_t *cb;
+	ulong_t remainder = iv_len;
+	ulong_t processed = 0;
+	uint8_t *datap, *ghash;
+	uint64_t len_a_len_c[2];
+
+	ghash = (uint8_t *)ctx->gcm_ghash;
+	cb = (uint8_t *)ctx->gcm_cb;
+	if (iv_len == 12) {
+		bcopy(iv, cb, 12);
+		cb[12] = 0;
+		cb[13] = 0;
+		cb[14] = 0;
+		cb[15] = 1;
+		/* J0 will be used again in the final */
+		copy_block(cb, (uint8_t *)ctx->gcm_J0);
+	} else {
+		/* GHASH the IV */
+		do {
+			if (remainder < block_size) {
+				bzero(cb, block_size);
+				bcopy(&(iv[processed]), cb, remainder);
+				datap = (uint8_t *)cb;
+				remainder = 0;
+			} else {
+				datap = (uint8_t *)(&(iv[processed]));
+				processed += block_size;
+				remainder -= block_size;
+			}
+			GHASH(ctx, datap, ghash);
+		} while (remainder > 0);
+
+		len_a_len_c[0] = 0;
+		len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(iv_len));
+		GHASH(ctx, len_a_len_c, ctx->gcm_J0);
+
+		/* J0 will be used again in the final */
+		copy_block((uint8_t *)ctx->gcm_J0, (uint8_t *)cb);
+	}
+}
+
+/*
+ * The following function is called at encrypt or decrypt init time
+ * for AES GCM mode.
+ */
+int
+gcm_init(gcm_ctx_t *ctx, unsigned char *iv, size_t iv_len,
+    unsigned char *auth_data, size_t auth_data_len, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	uint8_t *ghash, *datap, *authp;
+	size_t remainder, processed;
+
+	/* encrypt zero block to get subkey H */
+	bzero(ctx->gcm_H, sizeof (ctx->gcm_H));
+	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_H,
+	    (uint8_t *)ctx->gcm_H);
+
+	gcm_format_initial_blocks(iv, iv_len, ctx, block_size,
+	    copy_block, xor_block);
+
+	authp = (uint8_t *)ctx->gcm_tmp;
+	ghash = (uint8_t *)ctx->gcm_ghash;
+	bzero(authp, block_size);
+	bzero(ghash, block_size);
+
+	processed = 0;
+	remainder = auth_data_len;
+	do {
+		if (remainder < block_size) {
+			/*
+			 * There's not a block full of data, pad rest of
+			 * buffer with zero
+			 */
+			bzero(authp, block_size);
+			bcopy(&(auth_data[processed]), authp, remainder);
+			datap = (uint8_t *)authp;
+			remainder = 0;
+		} else {
+			datap = (uint8_t *)(&(auth_data[processed]));
+			processed += block_size;
+			remainder -= block_size;
+		}
+
+		/* add auth data to the hash */
+		GHASH(ctx, datap, ghash);
+
+	} while (remainder > 0);
+
+	return (CRYPTO_SUCCESS);
+}
+
+int
+gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	int rv;
+	CK_AES_GCM_PARAMS *gcm_param;
+
+	if (param != NULL) {
+		gcm_param = (CK_AES_GCM_PARAMS *)(void *)param;
+
+		if ((rv = gcm_validate_args(gcm_param)) != 0) {
+			return (rv);
+		}
+
+		gcm_ctx->gcm_tag_len = gcm_param->ulTagBits;
+		gcm_ctx->gcm_tag_len >>= 3;
+		gcm_ctx->gcm_processed_data_len = 0;
+
+		/* these values are in bits */
+		gcm_ctx->gcm_len_a_len_c[0]
+		    = htonll(CRYPTO_BYTES2BITS(gcm_param->ulAADLen));
+
+		rv = CRYPTO_SUCCESS;
+		gcm_ctx->gcm_flags |= GCM_MODE;
+	} else {
+		rv = CRYPTO_MECHANISM_PARAM_INVALID;
+		goto out;
+	}
+
+	if (gcm_init(gcm_ctx, gcm_param->pIv, gcm_param->ulIvLen,
+	    gcm_param->pAAD, gcm_param->ulAADLen, block_size,
+	    encrypt_block, copy_block, xor_block) != 0) {
+		rv = CRYPTO_MECHANISM_PARAM_INVALID;
+	}
+out:
+	return (rv);
+}
+
+int
+gmac_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(uint8_t *, uint8_t *),
+    void (*xor_block)(uint8_t *, uint8_t *))
+{
+	int rv;
+	CK_AES_GMAC_PARAMS *gmac_param;
+
+	if (param != NULL) {
+		gmac_param = (CK_AES_GMAC_PARAMS *)(void *)param;
+
+		gcm_ctx->gcm_tag_len = CRYPTO_BITS2BYTES(AES_GMAC_TAG_BITS);
+		gcm_ctx->gcm_processed_data_len = 0;
+
+		/* these values are in bits */
+		gcm_ctx->gcm_len_a_len_c[0]
+		    = htonll(CRYPTO_BYTES2BITS(gmac_param->ulAADLen));
+
+		rv = CRYPTO_SUCCESS;
+		gcm_ctx->gcm_flags |= GMAC_MODE;
+	} else {
+		rv = CRYPTO_MECHANISM_PARAM_INVALID;
+		goto out;
+	}
+
+	if (gcm_init(gcm_ctx, gmac_param->pIv, AES_GMAC_IV_LEN,
+	    gmac_param->pAAD, gmac_param->ulAADLen, block_size,
+	    encrypt_block, copy_block, xor_block) != 0) {
+		rv = CRYPTO_MECHANISM_PARAM_INVALID;
+	}
+out:
+	return (rv);
+}
+
+void *
+gcm_alloc_ctx(int kmflag)
+{
+	gcm_ctx_t *gcm_ctx;
+
+	if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL)
+		return (NULL);
+
+	gcm_ctx->gcm_flags = GCM_MODE;
+	return (gcm_ctx);
+}
+
+void *
+gmac_alloc_ctx(int kmflag)
+{
+	gcm_ctx_t *gcm_ctx;
+
+	if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL)
+		return (NULL);
+
+	gcm_ctx->gcm_flags = GMAC_MODE;
+	return (gcm_ctx);
+}
+
+void
+gcm_set_kmflag(gcm_ctx_t *ctx, int kmflag)
+{
+	ctx->gcm_kmflag = kmflag;
+}
+
+
+#ifdef __amd64
+
+#define	INTEL_PCLMULQDQ_FLAG (1 << 1)
+
+/*
+ * Return 1 if executing on Intel with PCLMULQDQ instructions,
+ * otherwise 0 (i.e., Intel without PCLMULQDQ or AMD64).
+ * Cache the result, as the CPU can't change.
+ *
+ * Note: the userland version uses getisax().  The kernel version uses
+ * is_x86_featureset().
+ */
+static int
+intel_pclmulqdq_instruction_present(void)
+{
+	static int cached_result = -1;
+	unsigned eax, ebx, ecx, edx;
+	unsigned func, subfunc;
+
+	if (cached_result == -1) { /* first time */
+		/* check for an intel cpu */
+		func = 0;
+		subfunc = 0;
+
+		__asm__ __volatile__(
+		    "cpuid"
+		    : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx)
+		    : "a"(func), "c"(subfunc));
+
+		if (memcmp((char *) (&ebx), "Genu", 4) == 0 &&
+		    memcmp((char *) (&edx), "ineI", 4) == 0 &&
+			memcmp((char *) (&ecx), "ntel", 4) == 0) {
+
+			func = 1;
+			subfunc = 0;
+
+			/* check for aes-ni instruction set */
+			__asm__ __volatile__(
+				"cpuid"
+				: "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx)
+				: "a"(func), "c"(subfunc));
+
+			cached_result = !!(ecx & INTEL_PCLMULQDQ_FLAG);
+		} else {
+			cached_result = 0;
+		}
+	}
+
+	return (cached_result);
+}
+
+#endif	/* __amd64 */

module/icp/algs/modes/modes.c

@@ -0,0 +1,159 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/zfs_context.h>
+#include <modes/modes.h>
+#include <sys/crypto/common.h>
+#include <sys/crypto/impl.h>
+
+/*
+ * Initialize by setting iov_or_mp to point to the current iovec or mp,
+ * and by setting current_offset to an offset within the current iovec or mp.
+ */
+void
+crypto_init_ptrs(crypto_data_t *out, void **iov_or_mp, offset_t *current_offset)
+{
+	offset_t offset;
+
+	switch (out->cd_format) {
+	case CRYPTO_DATA_RAW:
+		*current_offset = out->cd_offset;
+		break;
+
+	case CRYPTO_DATA_UIO: {
+		uio_t *uiop = out->cd_uio;
+		uintptr_t vec_idx;
+
+		offset = out->cd_offset;
+		for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
+		    offset >= uiop->uio_iov[vec_idx].iov_len;
+		    offset -= uiop->uio_iov[vec_idx++].iov_len)
+			;
+
+		*current_offset = offset;
+		*iov_or_mp = (void *)vec_idx;
+		break;
+	}
+	} /* end switch */
+}
+
+/*
+ * Get pointers for where in the output to copy a block of encrypted or
+ * decrypted data.  The iov_or_mp argument stores a pointer to the current
+ * iovec or mp, and offset stores an offset into the current iovec or mp.
+ */
+void
+crypto_get_ptrs(crypto_data_t *out, void **iov_or_mp, offset_t *current_offset,
+    uint8_t **out_data_1, size_t *out_data_1_len, uint8_t **out_data_2,
+    size_t amt)
+{
+	offset_t offset;
+
+	switch (out->cd_format) {
+	case CRYPTO_DATA_RAW: {
+		iovec_t *iov;
+
+		offset = *current_offset;
+		iov = &out->cd_raw;
+		if ((offset + amt) <= iov->iov_len) {
+			/* one block fits */
+			*out_data_1 = (uint8_t *)iov->iov_base + offset;
+			*out_data_1_len = amt;
+			*out_data_2 = NULL;
+			*current_offset = offset + amt;
+		}
+		break;
+	}
+
+	case CRYPTO_DATA_UIO: {
+		uio_t *uio = out->cd_uio;
+		iovec_t *iov;
+		offset_t offset;
+		uintptr_t vec_idx;
+		uint8_t *p;
+
+		offset = *current_offset;
+		vec_idx = (uintptr_t)(*iov_or_mp);
+		iov = (iovec_t *)&uio->uio_iov[vec_idx];
+		p = (uint8_t *)iov->iov_base + offset;
+		*out_data_1 = p;
+
+		if (offset + amt <= iov->iov_len) {
+			/* can fit one block into this iov */
+			*out_data_1_len = amt;
+			*out_data_2 = NULL;
+			*current_offset = offset + amt;
+		} else {
+			/* one block spans two iovecs */
+			*out_data_1_len = iov->iov_len - offset;
+			if (vec_idx == uio->uio_iovcnt)
+				return;
+			vec_idx++;
+			iov = (iovec_t *)&uio->uio_iov[vec_idx];
+			*out_data_2 = (uint8_t *)iov->iov_base;
+			*current_offset = amt - *out_data_1_len;
+		}
+		*iov_or_mp = (void *)vec_idx;
+		break;
+	}
+	} /* end switch */
+}
+
+void
+crypto_free_mode_ctx(void *ctx)
+{
+	common_ctx_t *common_ctx = (common_ctx_t *)ctx;
+
+	switch (common_ctx->cc_flags &
+	    (ECB_MODE|CBC_MODE|CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) {
+	case ECB_MODE:
+		kmem_free(common_ctx, sizeof (ecb_ctx_t));
+		break;
+
+	case CBC_MODE:
+		kmem_free(common_ctx, sizeof (cbc_ctx_t));
+		break;
+
+	case CTR_MODE:
+		kmem_free(common_ctx, sizeof (ctr_ctx_t));
+		break;
+
+	case CCM_MODE:
+		if (((ccm_ctx_t *)ctx)->ccm_pt_buf != NULL)
+			vmem_free(((ccm_ctx_t *)ctx)->ccm_pt_buf,
+			    ((ccm_ctx_t *)ctx)->ccm_data_len);
+
+		kmem_free(ctx, sizeof (ccm_ctx_t));
+		break;
+
+	case GCM_MODE:
+	case GMAC_MODE:
+		if (((gcm_ctx_t *)ctx)->gcm_pt_buf != NULL)
+			vmem_free(((gcm_ctx_t *)ctx)->gcm_pt_buf,
+			    ((gcm_ctx_t *)ctx)->gcm_pt_buf_len);
+
+		kmem_free(ctx, sizeof (gcm_ctx_t));
+	}
+}

module/icp/algs/sha1/sha1.c

@@ -0,0 +1,663 @@
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * The basic framework for this code came from the reference
+ * implementation for MD5.  That implementation is Copyright (C)
+ * 1991-2, RSA Data Security, Inc. Created 1991. All rights reserved.
+ *
+ * License to copy and use this software is granted provided that it
+ * is identified as the "RSA Data Security, Inc. MD5 Message-Digest
+ * Algorithm" in all material mentioning or referencing this software
+ * or this function.
+ *
+ * License is also granted to make and use derivative works provided
+ * that such works are identified as "derived from the RSA Data
+ * Security, Inc. MD5 Message-Digest Algorithm" in all material
+ * mentioning or referencing the derived work.
+ *
+ * RSA Data Security, Inc. makes no representations concerning either
+ * the merchantability of this software or the suitability of this
+ * software for any particular purpose. It is provided "as is"
+ * without express or implied warranty of any kind.
+ *
+ * These notices must be retained in any copies of any part of this
+ * documentation and/or software.
+ *
+ * NOTE: Cleaned-up and optimized, version of SHA1, based on the FIPS 180-1
+ * standard, available at http://www.itl.nist.gov/fipspubs/fip180-1.htm
+ * Not as fast as one would like -- further optimizations are encouraged
+ * and appreciated.
+ */
+
+#include <sys/zfs_context.h>
+#include <sha1/sha1.h>
+#include <sha1/sha1_consts.h>
+
+#ifdef _LITTLE_ENDIAN
+#include <sys/byteorder.h>
+#define	HAVE_HTONL
+#endif
+
+#define	_RESTRICT_KYWD
+
+static void Encode(uint8_t *, const uint32_t *, size_t);
+
+#if	defined(__amd64)
+
+#define	SHA1_TRANSFORM(ctx, in) sha1_block_data_order((ctx), (in), 1)
+#define	SHA1_TRANSFORM_BLOCKS(ctx, in, num) sha1_block_data_order((ctx), \
+		(in), (num))
+
+void sha1_block_data_order(SHA1_CTX *ctx, const void *inpp, size_t num_blocks);
+
+#else
+
+#define	SHA1_TRANSFORM(ctx, in) SHA1Transform((ctx), (in))
+
+static void SHA1Transform(SHA1_CTX *, const uint8_t *);
+
+#endif
+
+
+static uint8_t PADDING[64] = { 0x80, /* all zeros */ };
+
+/*
+ * F, G, and H are the basic SHA1 functions.
+ */
+#define	F(b, c, d)	(((b) & (c)) | ((~b) & (d)))
+#define	G(b, c, d)	((b) ^ (c) ^ (d))
+#define	H(b, c, d)	(((b) & (c)) | (((b)|(c)) & (d)))
+
+/*
+ * ROTATE_LEFT rotates x left n bits.
+ */
+
+#if	defined(__GNUC__) && defined(_LP64)
+static __inline__ uint64_t
+ROTATE_LEFT(uint64_t value, uint32_t n)
+{
+	uint32_t t32;
+
+	t32 = (uint32_t)value;
+	return ((t32 << n) | (t32 >> (32 - n)));
+}
+
+#else
+
+#define	ROTATE_LEFT(x, n)	\
+	(((x) << (n)) | ((x) >> ((sizeof (x) * NBBY)-(n))))
+
+#endif
+
+
+/*
+ * SHA1Init()
+ *
+ * purpose: initializes the sha1 context and begins and sha1 digest operation
+ *   input: SHA1_CTX *	: the context to initializes.
+ *  output: void
+ */
+
+void
+SHA1Init(SHA1_CTX *ctx)
+{
+	ctx->count[0] = ctx->count[1] = 0;
+
+	/*
+	 * load magic initialization constants. Tell lint
+	 * that these constants are unsigned by using U.
+	 */
+
+	ctx->state[0] = 0x67452301U;
+	ctx->state[1] = 0xefcdab89U;
+	ctx->state[2] = 0x98badcfeU;
+	ctx->state[3] = 0x10325476U;
+	ctx->state[4] = 0xc3d2e1f0U;
+}
+
+void
+SHA1Update(SHA1_CTX *ctx, const void *inptr, size_t input_len)
+{
+	uint32_t i, buf_index, buf_len;
+	const uint8_t *input = inptr;
+#if defined(__amd64)
+	uint32_t	block_count;
+#endif	/* __amd64 */
+
+	/* check for noop */
+	if (input_len == 0)
+		return;
+
+	/* compute number of bytes mod 64 */
+	buf_index = (ctx->count[1] >> 3) & 0x3F;
+
+	/* update number of bits */
+	if ((ctx->count[1] += (input_len << 3)) < (input_len << 3))
+		ctx->count[0]++;
+
+	ctx->count[0] += (input_len >> 29);
+
+	buf_len = 64 - buf_index;
+
+	/* transform as many times as possible */
+	i = 0;
+	if (input_len >= buf_len) {
+
+		/*
+		 * general optimization:
+		 *
+		 * only do initial bcopy() and SHA1Transform() if
+		 * buf_index != 0.  if buf_index == 0, we're just
+		 * wasting our time doing the bcopy() since there
+		 * wasn't any data left over from a previous call to
+		 * SHA1Update().
+		 */
+
+		if (buf_index) {
+			bcopy(input, &ctx->buf_un.buf8[buf_index], buf_len);
+			SHA1_TRANSFORM(ctx, ctx->buf_un.buf8);
+			i = buf_len;
+		}
+
+#if !defined(__amd64)
+		for (; i + 63 < input_len; i += 64)
+			SHA1_TRANSFORM(ctx, &input[i]);
+#else
+		block_count = (input_len - i) >> 6;
+		if (block_count > 0) {
+			SHA1_TRANSFORM_BLOCKS(ctx, &input[i], block_count);
+			i += block_count << 6;
+		}
+#endif	/* !__amd64 */
+
+		/*
+		 * general optimization:
+		 *
+		 * if i and input_len are the same, return now instead
+		 * of calling bcopy(), since the bcopy() in this case
+		 * will be an expensive nop.
+		 */
+
+		if (input_len == i)
+			return;
+
+		buf_index = 0;
+	}
+
+	/* buffer remaining input */
+	bcopy(&input[i], &ctx->buf_un.buf8[buf_index], input_len - i);
+}
+
+/*
+ * SHA1Final()
+ *
+ * purpose: ends an sha1 digest operation, finalizing the message digest and
+ *          zeroing the context.
+ *   input: uchar_t *	: A buffer to store the digest.
+ *			: The function actually uses void* because many
+ *			: callers pass things other than uchar_t here.
+ *          SHA1_CTX *  : the context to finalize, save, and zero
+ *  output: void
+ */
+
+void
+SHA1Final(void *digest, SHA1_CTX *ctx)
+{
+	uint8_t		bitcount_be[sizeof (ctx->count)];
+	uint32_t	index = (ctx->count[1] >> 3) & 0x3f;
+
+	/* store bit count, big endian */
+	Encode(bitcount_be, ctx->count, sizeof (bitcount_be));
+
+	/* pad out to 56 mod 64 */
+	SHA1Update(ctx, PADDING, ((index < 56) ? 56 : 120) - index);
+
+	/* append length (before padding) */
+	SHA1Update(ctx, bitcount_be, sizeof (bitcount_be));
+
+	/* store state in digest */
+	Encode(digest, ctx->state, sizeof (ctx->state));
+
+	/* zeroize sensitive information */
+	bzero(ctx, sizeof (*ctx));
+}
+
+
+#if !defined(__amd64)
+
+typedef uint32_t sha1word;
+
+/*
+ * sparc optimization:
+ *
+ * on the sparc, we can load big endian 32-bit data easily.  note that
+ * special care must be taken to ensure the address is 32-bit aligned.
+ * in the interest of speed, we don't check to make sure, since
+ * careful programming can guarantee this for us.
+ */
+
+#if	defined(_BIG_ENDIAN)
+#define	LOAD_BIG_32(addr)	(*(uint32_t *)(addr))
+
+#elif	defined(HAVE_HTONL)
+#define	LOAD_BIG_32(addr) htonl(*((uint32_t *)(addr)))
+
+#else
+/* little endian -- will work on big endian, but slowly */
+#define	LOAD_BIG_32(addr)	\
+	(((addr)[0] << 24) | ((addr)[1] << 16) | ((addr)[2] << 8) | (addr)[3])
+#endif	/* _BIG_ENDIAN */
+
+/*
+ * SHA1Transform()
+ */
+#if	defined(W_ARRAY)
+#define	W(n) w[n]
+#else	/* !defined(W_ARRAY) */
+#define	W(n) w_ ## n
+#endif	/* !defined(W_ARRAY) */
+
+void /* CSTYLED */
+SHA1Transform(SHA1_CTX *ctx, const uint8_t blk[64])
+{
+	/* CSTYLED */
+	sha1word a = ctx->state[0];
+	sha1word b = ctx->state[1];
+	sha1word c = ctx->state[2];
+	sha1word d = ctx->state[3];
+	sha1word e = ctx->state[4];
+
+#if	defined(W_ARRAY)
+	sha1word	w[16];
+#else	/* !defined(W_ARRAY) */
+	sha1word	w_0, w_1, w_2,  w_3,  w_4,  w_5,  w_6,  w_7;
+	sha1word	w_8, w_9, w_10, w_11, w_12, w_13, w_14, w_15;
+#endif	/* !defined(W_ARRAY) */
+
+	W(0)  = LOAD_BIG_32((void *)(blk +  0));
+	W(1)  = LOAD_BIG_32((void *)(blk +  4));
+	W(2)  = LOAD_BIG_32((void *)(blk +  8));
+	W(3)  = LOAD_BIG_32((void *)(blk + 12));
+	W(4)  = LOAD_BIG_32((void *)(blk + 16));
+	W(5)  = LOAD_BIG_32((void *)(blk + 20));
+	W(6)  = LOAD_BIG_32((void *)(blk + 24));
+	W(7)  = LOAD_BIG_32((void *)(blk + 28));
+	W(8)  = LOAD_BIG_32((void *)(blk + 32));
+	W(9)  = LOAD_BIG_32((void *)(blk + 36));
+	W(10) = LOAD_BIG_32((void *)(blk + 40));
+	W(11) = LOAD_BIG_32((void *)(blk + 44));
+	W(12) = LOAD_BIG_32((void *)(blk + 48));
+	W(13) = LOAD_BIG_32((void *)(blk + 52));
+	W(14) = LOAD_BIG_32((void *)(blk + 56));
+	W(15) = LOAD_BIG_32((void *)(blk + 60));
+
+	/*
+	 * general optimization:
+	 *
+	 * even though this approach is described in the standard as
+	 * being slower algorithmically, it is 30-40% faster than the
+	 * "faster" version under SPARC, because this version has more
+	 * of the constraints specified at compile-time and uses fewer
+	 * variables (and therefore has better register utilization)
+	 * than its "speedier" brother.  (i've tried both, trust me)
+	 *
+	 * for either method given in the spec, there is an "assignment"
+	 * phase where the following takes place:
+	 *
+	 *	tmp = (main_computation);
+	 *	e = d; d = c; c = rotate_left(b, 30); b = a; a = tmp;
+	 *
+	 * we can make the algorithm go faster by not doing this work,
+	 * but just pretending that `d' is now `e', etc. this works
+	 * really well and obviates the need for a temporary variable.
+	 * however, we still explicitly perform the rotate action,
+	 * since it is cheaper on SPARC to do it once than to have to
+	 * do it over and over again.
+	 */
+
+	/* round 1 */
+	e = ROTATE_LEFT(a, 5) + F(b, c, d) + e + W(0) + SHA1_CONST(0); /* 0 */
+	b = ROTATE_LEFT(b, 30);
+
+	d = ROTATE_LEFT(e, 5) + F(a, b, c) + d + W(1) + SHA1_CONST(0); /* 1 */
+	a = ROTATE_LEFT(a, 30);
+
+	c = ROTATE_LEFT(d, 5) + F(e, a, b) + c + W(2) + SHA1_CONST(0); /* 2 */
+	e = ROTATE_LEFT(e, 30);
+
+	b = ROTATE_LEFT(c, 5) + F(d, e, a) + b + W(3) + SHA1_CONST(0); /* 3 */
+	d = ROTATE_LEFT(d, 30);
+
+	a = ROTATE_LEFT(b, 5) + F(c, d, e) + a + W(4) + SHA1_CONST(0); /* 4 */
+	c = ROTATE_LEFT(c, 30);
+
+	e = ROTATE_LEFT(a, 5) + F(b, c, d) + e + W(5) + SHA1_CONST(0); /* 5 */
+	b = ROTATE_LEFT(b, 30);
+
+	d = ROTATE_LEFT(e, 5) + F(a, b, c) + d + W(6) + SHA1_CONST(0); /* 6 */
+	a = ROTATE_LEFT(a, 30);
+
+	c = ROTATE_LEFT(d, 5) + F(e, a, b) + c + W(7) + SHA1_CONST(0); /* 7 */
+	e = ROTATE_LEFT(e, 30);
+
+	b = ROTATE_LEFT(c, 5) + F(d, e, a) + b + W(8) + SHA1_CONST(0); /* 8 */
+	d = ROTATE_LEFT(d, 30);
+
+	a = ROTATE_LEFT(b, 5) + F(c, d, e) + a + W(9) + SHA1_CONST(0); /* 9 */
+	c = ROTATE_LEFT(c, 30);
+
+	e = ROTATE_LEFT(a, 5) + F(b, c, d) + e + W(10) + SHA1_CONST(0); /* 10 */
+	b = ROTATE_LEFT(b, 30);
+
+	d = ROTATE_LEFT(e, 5) + F(a, b, c) + d + W(11) + SHA1_CONST(0); /* 11 */
+	a = ROTATE_LEFT(a, 30);
+
+	c = ROTATE_LEFT(d, 5) + F(e, a, b) + c + W(12) + SHA1_CONST(0); /* 12 */
+	e = ROTATE_LEFT(e, 30);
+
+	b = ROTATE_LEFT(c, 5) + F(d, e, a) + b + W(13) + SHA1_CONST(0); /* 13 */
+	d = ROTATE_LEFT(d, 30);
+
+	a = ROTATE_LEFT(b, 5) + F(c, d, e) + a + W(14) + SHA1_CONST(0); /* 14 */
+	c = ROTATE_LEFT(c, 30);
+
+	e = ROTATE_LEFT(a, 5) + F(b, c, d) + e + W(15) + SHA1_CONST(0); /* 15 */
+	b = ROTATE_LEFT(b, 30);
+
+	W(0) = ROTATE_LEFT((W(13) ^ W(8) ^ W(2) ^ W(0)), 1);		/* 16 */
+	d = ROTATE_LEFT(e, 5) + F(a, b, c) + d + W(0) + SHA1_CONST(0);
+	a = ROTATE_LEFT(a, 30);
+
+	W(1) = ROTATE_LEFT((W(14) ^ W(9) ^ W(3) ^ W(1)), 1);		/* 17 */
+	c = ROTATE_LEFT(d, 5) + F(e, a, b) + c + W(1) + SHA1_CONST(0);
+	e = ROTATE_LEFT(e, 30);
+
+	W(2) = ROTATE_LEFT((W(15) ^ W(10) ^ W(4) ^ W(2)), 1);	/* 18 */
+	b = ROTATE_LEFT(c, 5) + F(d, e, a) + b + W(2) + SHA1_CONST(0);
+	d = ROTATE_LEFT(d, 30);
+
+	W(3) = ROTATE_LEFT((W(0) ^ W(11) ^ W(5) ^ W(3)), 1);		/* 19 */
+	a = ROTATE_LEFT(b, 5) + F(c, d, e) + a + W(3) + SHA1_CONST(0);
+	c = ROTATE_LEFT(c, 30);
+
+	/* round 2 */
+	W(4) = ROTATE_LEFT((W(1) ^ W(12) ^ W(6) ^ W(4)), 1);		/* 20 */
+	e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(4) + SHA1_CONST(1);
+	b = ROTATE_LEFT(b, 30);
+
+	W(5) = ROTATE_LEFT((W(2) ^ W(13) ^ W(7) ^ W(5)), 1);		/* 21 */
+	d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(5) + SHA1_CONST(1);
+	a = ROTATE_LEFT(a, 30);
+
+	W(6) = ROTATE_LEFT((W(3) ^ W(14) ^ W(8) ^ W(6)), 1);		/* 22 */
+	c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(6) + SHA1_CONST(1);
+	e = ROTATE_LEFT(e, 30);
+
+	W(7) = ROTATE_LEFT((W(4) ^ W(15) ^ W(9) ^ W(7)), 1);		/* 23 */
+	b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(7) + SHA1_CONST(1);
+	d = ROTATE_LEFT(d, 30);
+
+	W(8) = ROTATE_LEFT((W(5) ^ W(0) ^ W(10) ^ W(8)), 1);		/* 24 */
+	a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(8) + SHA1_CONST(1);
+	c = ROTATE_LEFT(c, 30);
+
+	W(9) = ROTATE_LEFT((W(6) ^ W(1) ^ W(11) ^ W(9)), 1);		/* 25 */
+	e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(9) + SHA1_CONST(1);
+	b = ROTATE_LEFT(b, 30);
+
+	W(10) = ROTATE_LEFT((W(7) ^ W(2) ^ W(12) ^ W(10)), 1);	/* 26 */
+	d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(10) + SHA1_CONST(1);
+	a = ROTATE_LEFT(a, 30);
+
+	W(11) = ROTATE_LEFT((W(8) ^ W(3) ^ W(13) ^ W(11)), 1);	/* 27 */
+	c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(11) + SHA1_CONST(1);
+	e = ROTATE_LEFT(e, 30);
+
+	W(12) = ROTATE_LEFT((W(9) ^ W(4) ^ W(14) ^ W(12)), 1);	/* 28 */
+	b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(12) + SHA1_CONST(1);
+	d = ROTATE_LEFT(d, 30);
+
+	W(13) = ROTATE_LEFT((W(10) ^ W(5) ^ W(15) ^ W(13)), 1);	/* 29 */
+	a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(13) + SHA1_CONST(1);
+	c = ROTATE_LEFT(c, 30);
+
+	W(14) = ROTATE_LEFT((W(11) ^ W(6) ^ W(0) ^ W(14)), 1);	/* 30 */
+	e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(14) + SHA1_CONST(1);
+	b = ROTATE_LEFT(b, 30);
+
+	W(15) = ROTATE_LEFT((W(12) ^ W(7) ^ W(1) ^ W(15)), 1);	/* 31 */
+	d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(15) + SHA1_CONST(1);
+	a = ROTATE_LEFT(a, 30);
+
+	W(0) = ROTATE_LEFT((W(13) ^ W(8) ^ W(2) ^ W(0)), 1);		/* 32 */
+	c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(0) + SHA1_CONST(1);
+	e = ROTATE_LEFT(e, 30);
+
+	W(1) = ROTATE_LEFT((W(14) ^ W(9) ^ W(3) ^ W(1)), 1);		/* 33 */
+	b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(1) + SHA1_CONST(1);
+	d = ROTATE_LEFT(d, 30);
+
+	W(2) = ROTATE_LEFT((W(15) ^ W(10) ^ W(4) ^ W(2)), 1);	/* 34 */
+	a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(2) + SHA1_CONST(1);
+	c = ROTATE_LEFT(c, 30);
+
+	W(3) = ROTATE_LEFT((W(0) ^ W(11) ^ W(5) ^ W(3)), 1);		/* 35 */
+	e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(3) + SHA1_CONST(1);
+	b = ROTATE_LEFT(b, 30);
+
+	W(4) = ROTATE_LEFT((W(1) ^ W(12) ^ W(6) ^ W(4)), 1);		/* 36 */
+	d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(4) + SHA1_CONST(1);
+	a = ROTATE_LEFT(a, 30);
+
+	W(5) = ROTATE_LEFT((W(2) ^ W(13) ^ W(7) ^ W(5)), 1);		/* 37 */
+	c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(5) + SHA1_CONST(1);
+	e = ROTATE_LEFT(e, 30);
+
+	W(6) = ROTATE_LEFT((W(3) ^ W(14) ^ W(8) ^ W(6)), 1);		/* 38 */
+	b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(6) + SHA1_CONST(1);
+	d = ROTATE_LEFT(d, 30);
+
+	W(7) = ROTATE_LEFT((W(4) ^ W(15) ^ W(9) ^ W(7)), 1);		/* 39 */
+	a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(7) + SHA1_CONST(1);
+	c = ROTATE_LEFT(c, 30);
+
+	/* round 3 */
+	W(8) = ROTATE_LEFT((W(5) ^ W(0) ^ W(10) ^ W(8)), 1);		/* 40 */
+	e = ROTATE_LEFT(a, 5) + H(b, c, d) + e + W(8) + SHA1_CONST(2);
+	b = ROTATE_LEFT(b, 30);
+
+	W(9) = ROTATE_LEFT((W(6) ^ W(1) ^ W(11) ^ W(9)), 1);		/* 41 */
+	d = ROTATE_LEFT(e, 5) + H(a, b, c) + d + W(9) + SHA1_CONST(2);
+	a = ROTATE_LEFT(a, 30);
+
+	W(10) = ROTATE_LEFT((W(7) ^ W(2) ^ W(12) ^ W(10)), 1);	/* 42 */
+	c = ROTATE_LEFT(d, 5) + H(e, a, b) + c + W(10) + SHA1_CONST(2);
+	e = ROTATE_LEFT(e, 30);
+
+	W(11) = ROTATE_LEFT((W(8) ^ W(3) ^ W(13) ^ W(11)), 1);	/* 43 */
+	b = ROTATE_LEFT(c, 5) + H(d, e, a) + b + W(11) + SHA1_CONST(2);
+	d = ROTATE_LEFT(d, 30);
+
+	W(12) = ROTATE_LEFT((W(9) ^ W(4) ^ W(14) ^ W(12)), 1);	/* 44 */
+	a = ROTATE_LEFT(b, 5) + H(c, d, e) + a + W(12) + SHA1_CONST(2);
+	c = ROTATE_LEFT(c, 30);
+
+	W(13) = ROTATE_LEFT((W(10) ^ W(5) ^ W(15) ^ W(13)), 1);	/* 45 */
+	e = ROTATE_LEFT(a, 5) + H(b, c, d) + e + W(13) + SHA1_CONST(2);
+	b = ROTATE_LEFT(b, 30);
+
+	W(14) = ROTATE_LEFT((W(11) ^ W(6) ^ W(0) ^ W(14)), 1);	/* 46 */
+	d = ROTATE_LEFT(e, 5) + H(a, b, c) + d + W(14) + SHA1_CONST(2);
+	a = ROTATE_LEFT(a, 30);
+
+	W(15) = ROTATE_LEFT((W(12) ^ W(7) ^ W(1) ^ W(15)), 1);	/* 47 */
+	c = ROTATE_LEFT(d, 5) + H(e, a, b) + c + W(15) + SHA1_CONST(2);
+	e = ROTATE_LEFT(e, 30);
+
+	W(0) = ROTATE_LEFT((W(13) ^ W(8) ^ W(2) ^ W(0)), 1);		/* 48 */
+	b = ROTATE_LEFT(c, 5) + H(d, e, a) + b + W(0) + SHA1_CONST(2);
+	d = ROTATE_LEFT(d, 30);
+
+	W(1) = ROTATE_LEFT((W(14) ^ W(9) ^ W(3) ^ W(1)), 1);		/* 49 */
+	a = ROTATE_LEFT(b, 5) + H(c, d, e) + a + W(1) + SHA1_CONST(2);
+	c = ROTATE_LEFT(c, 30);
+
+	W(2) = ROTATE_LEFT((W(15) ^ W(10) ^ W(4) ^ W(2)), 1);	/* 50 */
+	e = ROTATE_LEFT(a, 5) + H(b, c, d) + e + W(2) + SHA1_CONST(2);
+	b = ROTATE_LEFT(b, 30);
+
+	W(3) = ROTATE_LEFT((W(0) ^ W(11) ^ W(5) ^ W(3)), 1);		/* 51 */
+	d = ROTATE_LEFT(e, 5) + H(a, b, c) + d + W(3) + SHA1_CONST(2);
+	a = ROTATE_LEFT(a, 30);
+
+	W(4) = ROTATE_LEFT((W(1) ^ W(12) ^ W(6) ^ W(4)), 1);		/* 52 */
+	c = ROTATE_LEFT(d, 5) + H(e, a, b) + c + W(4) + SHA1_CONST(2);
+	e = ROTATE_LEFT(e, 30);
+
+	W(5) = ROTATE_LEFT((W(2) ^ W(13) ^ W(7) ^ W(5)), 1);		/* 53 */
+	b = ROTATE_LEFT(c, 5) + H(d, e, a) + b + W(5) + SHA1_CONST(2);
+	d = ROTATE_LEFT(d, 30);
+
+	W(6) = ROTATE_LEFT((W(3) ^ W(14) ^ W(8) ^ W(6)), 1);		/* 54 */
+	a = ROTATE_LEFT(b, 5) + H(c, d, e) + a + W(6) + SHA1_CONST(2);
+	c = ROTATE_LEFT(c, 30);
+
+	W(7) = ROTATE_LEFT((W(4) ^ W(15) ^ W(9) ^ W(7)), 1);		/* 55 */
+	e = ROTATE_LEFT(a, 5) + H(b, c, d) + e + W(7) + SHA1_CONST(2);
+	b = ROTATE_LEFT(b, 30);
+
+	W(8) = ROTATE_LEFT((W(5) ^ W(0) ^ W(10) ^ W(8)), 1);		/* 56 */
+	d = ROTATE_LEFT(e, 5) + H(a, b, c) + d + W(8) + SHA1_CONST(2);
+	a = ROTATE_LEFT(a, 30);
+
+	W(9) = ROTATE_LEFT((W(6) ^ W(1) ^ W(11) ^ W(9)), 1);		/* 57 */
+	c = ROTATE_LEFT(d, 5) + H(e, a, b) + c + W(9) + SHA1_CONST(2);
+	e = ROTATE_LEFT(e, 30);
+
+	W(10) = ROTATE_LEFT((W(7) ^ W(2) ^ W(12) ^ W(10)), 1);	/* 58 */
+	b = ROTATE_LEFT(c, 5) + H(d, e, a) + b + W(10) + SHA1_CONST(2);
+	d = ROTATE_LEFT(d, 30);
+
+	W(11) = ROTATE_LEFT((W(8) ^ W(3) ^ W(13) ^ W(11)), 1);	/* 59 */
+	a = ROTATE_LEFT(b, 5) + H(c, d, e) + a + W(11) + SHA1_CONST(2);
+	c = ROTATE_LEFT(c, 30);
+
+	/* round 4 */
+	W(12) = ROTATE_LEFT((W(9) ^ W(4) ^ W(14) ^ W(12)), 1);	/* 60 */
+	e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(12) + SHA1_CONST(3);
+	b = ROTATE_LEFT(b, 30);
+
+	W(13) = ROTATE_LEFT((W(10) ^ W(5) ^ W(15) ^ W(13)), 1);	/* 61 */
+	d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(13) + SHA1_CONST(3);
+	a = ROTATE_LEFT(a, 30);
+
+	W(14) = ROTATE_LEFT((W(11) ^ W(6) ^ W(0) ^ W(14)), 1);	/* 62 */
+	c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(14) + SHA1_CONST(3);
+	e = ROTATE_LEFT(e, 30);
+
+	W(15) = ROTATE_LEFT((W(12) ^ W(7) ^ W(1) ^ W(15)), 1);	/* 63 */
+	b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(15) + SHA1_CONST(3);
+	d = ROTATE_LEFT(d, 30);
+
+	W(0) = ROTATE_LEFT((W(13) ^ W(8) ^ W(2) ^ W(0)), 1);		/* 64 */
+	a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(0) + SHA1_CONST(3);
+	c = ROTATE_LEFT(c, 30);
+
+	W(1) = ROTATE_LEFT((W(14) ^ W(9) ^ W(3) ^ W(1)), 1);		/* 65 */
+	e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(1) + SHA1_CONST(3);
+	b = ROTATE_LEFT(b, 30);
+
+	W(2) = ROTATE_LEFT((W(15) ^ W(10) ^ W(4) ^ W(2)), 1);	/* 66 */
+	d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(2) + SHA1_CONST(3);
+	a = ROTATE_LEFT(a, 30);
+
+	W(3) = ROTATE_LEFT((W(0) ^ W(11) ^ W(5) ^ W(3)), 1);		/* 67 */
+	c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(3) + SHA1_CONST(3);
+	e = ROTATE_LEFT(e, 30);
+
+	W(4) = ROTATE_LEFT((W(1) ^ W(12) ^ W(6) ^ W(4)), 1);		/* 68 */
+	b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(4) + SHA1_CONST(3);
+	d = ROTATE_LEFT(d, 30);
+
+	W(5) = ROTATE_LEFT((W(2) ^ W(13) ^ W(7) ^ W(5)), 1);		/* 69 */
+	a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(5) + SHA1_CONST(3);
+	c = ROTATE_LEFT(c, 30);
+
+	W(6) = ROTATE_LEFT((W(3) ^ W(14) ^ W(8) ^ W(6)), 1);		/* 70 */
+	e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(6) + SHA1_CONST(3);
+	b = ROTATE_LEFT(b, 30);
+
+	W(7) = ROTATE_LEFT((W(4) ^ W(15) ^ W(9) ^ W(7)), 1);		/* 71 */
+	d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(7) + SHA1_CONST(3);
+	a = ROTATE_LEFT(a, 30);
+
+	W(8) = ROTATE_LEFT((W(5) ^ W(0) ^ W(10) ^ W(8)), 1);		/* 72 */
+	c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(8) + SHA1_CONST(3);
+	e = ROTATE_LEFT(e, 30);
+
+	W(9) = ROTATE_LEFT((W(6) ^ W(1) ^ W(11) ^ W(9)), 1);		/* 73 */
+	b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(9) + SHA1_CONST(3);
+	d = ROTATE_LEFT(d, 30);
+
+	W(10) = ROTATE_LEFT((W(7) ^ W(2) ^ W(12) ^ W(10)), 1);	/* 74 */
+	a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(10) + SHA1_CONST(3);
+	c = ROTATE_LEFT(c, 30);
+
+	W(11) = ROTATE_LEFT((W(8) ^ W(3) ^ W(13) ^ W(11)), 1);	/* 75 */
+	e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(11) + SHA1_CONST(3);
+	b = ROTATE_LEFT(b, 30);
+
+	W(12) = ROTATE_LEFT((W(9) ^ W(4) ^ W(14) ^ W(12)), 1);	/* 76 */
+	d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(12) + SHA1_CONST(3);
+	a = ROTATE_LEFT(a, 30);
+
+	W(13) = ROTATE_LEFT((W(10) ^ W(5) ^ W(15) ^ W(13)), 1);	/* 77 */
+	c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(13) + SHA1_CONST(3);
+	e = ROTATE_LEFT(e, 30);
+
+	W(14) = ROTATE_LEFT((W(11) ^ W(6) ^ W(0) ^ W(14)), 1);	/* 78 */
+	b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(14) + SHA1_CONST(3);
+	d = ROTATE_LEFT(d, 30);
+
+	W(15) = ROTATE_LEFT((W(12) ^ W(7) ^ W(1) ^ W(15)), 1);	/* 79 */
+
+	ctx->state[0] += ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(15) +
+	    SHA1_CONST(3);
+	ctx->state[1] += b;
+	ctx->state[2] += ROTATE_LEFT(c, 30);
+	ctx->state[3] += d;
+	ctx->state[4] += e;
+
+	/* zeroize sensitive information */
+	W(0) = W(1) = W(2) = W(3) = W(4) = W(5) = W(6) = W(7) = W(8) = 0;
+	W(9) = W(10) = W(11) = W(12) = W(13) = W(14) = W(15) = 0;
+}
+#endif	/* !__amd64 */
+
+
+/*
+ * Encode()
+ *
+ * purpose: to convert a list of numbers from little endian to big endian
+ *   input: uint8_t *	: place to store the converted big endian numbers
+ *	    uint32_t *	: place to get numbers to convert from
+ *          size_t	: the length of the input in bytes
+ *  output: void
+ */
+
+static void
+Encode(uint8_t *_RESTRICT_KYWD output, const uint32_t *_RESTRICT_KYWD input,
+    size_t len)
+{
+	size_t		i, j;
+
+	for (i = 0, j = 0; j < len; i++, j += 4) {
+		output[j]	= (input[i] >> 24) & 0xff;
+		output[j + 1]	= (input[i] >> 16) & 0xff;
+		output[j + 2]	= (input[i] >>  8) & 0xff;
+		output[j + 3]	= input[i] & 0xff;
+	}
+}

module/icp/algs/sha2/sha2.c

@@ -0,0 +1,495 @@
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+/*
+ * Copyright 2013 Saso Kiselkov.  All rights reserved.
+ */
+
+/*
+ * The basic framework for this code came from the reference
+ * implementation for MD5.  That implementation is Copyright (C)
+ * 1991-2, RSA Data Security, Inc. Created 1991. All rights reserved.
+ *
+ * License to copy and use this software is granted provided that it
+ * is identified as the "RSA Data Security, Inc. MD5 Message-Digest
+ * Algorithm" in all material mentioning or referencing this software
+ * or this function.
+ *
+ * License is also granted to make and use derivative works provided
+ * that such works are identified as "derived from the RSA Data
+ * Security, Inc. MD5 Message-Digest Algorithm" in all material
+ * mentioning or referencing the derived work.
+ *
+ * RSA Data Security, Inc. makes no representations concerning either
+ * the merchantability of this software or the suitability of this
+ * software for any particular purpose. It is provided "as is"
+ * without express or implied warranty of any kind.
+ *
+ * These notices must be retained in any copies of any part of this
+ * documentation and/or software.
+ *
+ * NOTE: Cleaned-up and optimized, version of SHA2, based on the FIPS 180-2
+ * standard, available at
+ * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2.pdf
+ * Not as fast as one would like -- further optimizations are encouraged
+ * and appreciated.
+ */
+
+#include <sys/zfs_context.h>
+#define	_SHA2_IMPL
+#include <sha2/sha2.h>
+#include <sha2/sha2_consts.h>
+
+#define	_RESTRICT_KYWD
+
+#ifdef _LITTLE_ENDIAN
+#include <sys/byteorder.h>
+#define	HAVE_HTONL
+#endif
+
+static void Encode(uint8_t *, uint32_t *, size_t);
+
+#if	defined(__amd64)
+#define	SHA256Transform(ctx, in) SHA256TransformBlocks((ctx), (in), 1)
+void SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num);
+#else
+static void SHA256Transform(SHA2_CTX *, const uint8_t *);
+#endif	/* __amd64 */
+
+static uint8_t PADDING[128] = { 0x80, /* all zeros */ };
+
+/* Ch and Maj are the basic SHA2 functions. */
+#define	Ch(b, c, d)	(((b) & (c)) ^ ((~b) & (d)))
+#define	Maj(b, c, d)	(((b) & (c)) ^ ((b) & (d)) ^ ((c) & (d)))
+
+/* Rotates x right n bits. */
+#define	ROTR(x, n)	\
+	(((x) >> (n)) | ((x) << ((sizeof (x) * NBBY)-(n))))
+
+/* Shift x right n bits */
+#define	SHR(x, n)	((x) >> (n))
+
+/* SHA256 Functions */
+#define	BIGSIGMA0_256(x)	(ROTR((x), 2) ^ ROTR((x), 13) ^ ROTR((x), 22))
+#define	BIGSIGMA1_256(x)	(ROTR((x), 6) ^ ROTR((x), 11) ^ ROTR((x), 25))
+#define	SIGMA0_256(x)		(ROTR((x), 7) ^ ROTR((x), 18) ^ SHR((x), 3))
+#define	SIGMA1_256(x)		(ROTR((x), 17) ^ ROTR((x), 19) ^ SHR((x), 10))
+
+#define	SHA256ROUND(a, b, c, d, e, f, g, h, i, w)			\
+	T1 = h + BIGSIGMA1_256(e) + Ch(e, f, g) + SHA256_CONST(i) + w;	\
+	d += T1;							\
+	T2 = BIGSIGMA0_256(a) + Maj(a, b, c);				\
+	h = T1 + T2
+
+/*
+ * sparc optimization:
+ *
+ * on the sparc, we can load big endian 32-bit data easily.  note that
+ * special care must be taken to ensure the address is 32-bit aligned.
+ * in the interest of speed, we don't check to make sure, since
+ * careful programming can guarantee this for us.
+ */
+
+#if	defined(_BIG_ENDIAN)
+#define	LOAD_BIG_32(addr)	(*(uint32_t *)(addr))
+#define	LOAD_BIG_64(addr)	(*(uint64_t *)(addr))
+
+#elif	defined(HAVE_HTONL)
+#define	LOAD_BIG_32(addr) htonl(*((uint32_t *)(addr)))
+#define	LOAD_BIG_64(addr) htonll(*((uint64_t *)(addr)))
+
+#else
+/* little endian -- will work on big endian, but slowly */
+#define	LOAD_BIG_32(addr)	\
+	(((addr)[0] << 24) | ((addr)[1] << 16) | ((addr)[2] << 8) | (addr)[3])
+#define	LOAD_BIG_64(addr)	\
+	(((uint64_t)(addr)[0] << 56) | ((uint64_t)(addr)[1] << 48) |	\
+	    ((uint64_t)(addr)[2] << 40) | ((uint64_t)(addr)[3] << 32) |	\
+	    ((uint64_t)(addr)[4] << 24) | ((uint64_t)(addr)[5] << 16) |	\
+	    ((uint64_t)(addr)[6] << 8) | (uint64_t)(addr)[7])
+#endif	/* _BIG_ENDIAN */
+
+
+#if	!defined(__amd64)
+/* SHA256 Transform */
+
+static void
+SHA256Transform(SHA2_CTX *ctx, const uint8_t *blk)
+{
+	uint32_t a = ctx->state.s32[0];
+	uint32_t b = ctx->state.s32[1];
+	uint32_t c = ctx->state.s32[2];
+	uint32_t d = ctx->state.s32[3];
+	uint32_t e = ctx->state.s32[4];
+	uint32_t f = ctx->state.s32[5];
+	uint32_t g = ctx->state.s32[6];
+	uint32_t h = ctx->state.s32[7];
+
+	uint32_t w0, w1, w2, w3, w4, w5, w6, w7;
+	uint32_t w8, w9, w10, w11, w12, w13, w14, w15;
+	uint32_t T1, T2;
+
+	if ((uintptr_t)blk & 0x3) {		/* not 4-byte aligned? */
+		bcopy(blk, ctx->buf_un.buf32,  sizeof (ctx->buf_un.buf32));
+		blk = (uint8_t *)ctx->buf_un.buf32;
+	}
+
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w0 =  LOAD_BIG_32(blk + 4 * 0);
+	SHA256ROUND(a, b, c, d, e, f, g, h, 0, w0);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w1 =  LOAD_BIG_32(blk + 4 * 1);
+	SHA256ROUND(h, a, b, c, d, e, f, g, 1, w1);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w2 =  LOAD_BIG_32(blk + 4 * 2);
+	SHA256ROUND(g, h, a, b, c, d, e, f, 2, w2);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w3 =  LOAD_BIG_32(blk + 4 * 3);
+	SHA256ROUND(f, g, h, a, b, c, d, e, 3, w3);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w4 =  LOAD_BIG_32(blk + 4 * 4);
+	SHA256ROUND(e, f, g, h, a, b, c, d, 4, w4);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w5 =  LOAD_BIG_32(blk + 4 * 5);
+	SHA256ROUND(d, e, f, g, h, a, b, c, 5, w5);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w6 =  LOAD_BIG_32(blk + 4 * 6);
+	SHA256ROUND(c, d, e, f, g, h, a, b, 6, w6);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w7 =  LOAD_BIG_32(blk + 4 * 7);
+	SHA256ROUND(b, c, d, e, f, g, h, a, 7, w7);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w8 =  LOAD_BIG_32(blk + 4 * 8);
+	SHA256ROUND(a, b, c, d, e, f, g, h, 8, w8);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w9 =  LOAD_BIG_32(blk + 4 * 9);
+	SHA256ROUND(h, a, b, c, d, e, f, g, 9, w9);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w10 =  LOAD_BIG_32(blk + 4 * 10);
+	SHA256ROUND(g, h, a, b, c, d, e, f, 10, w10);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w11 =  LOAD_BIG_32(blk + 4 * 11);
+	SHA256ROUND(f, g, h, a, b, c, d, e, 11, w11);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w12 =  LOAD_BIG_32(blk + 4 * 12);
+	SHA256ROUND(e, f, g, h, a, b, c, d, 12, w12);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w13 =  LOAD_BIG_32(blk + 4 * 13);
+	SHA256ROUND(d, e, f, g, h, a, b, c, 13, w13);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w14 =  LOAD_BIG_32(blk + 4 * 14);
+	SHA256ROUND(c, d, e, f, g, h, a, b, 14, w14);
+	/* LINTED E_BAD_PTR_CAST_ALIGN */
+	w15 =  LOAD_BIG_32(blk + 4 * 15);
+	SHA256ROUND(b, c, d, e, f, g, h, a, 15, w15);
+
+	w0 = SIGMA1_256(w14) + w9 + SIGMA0_256(w1) + w0;
+	SHA256ROUND(a, b, c, d, e, f, g, h, 16, w0);
+	w1 = SIGMA1_256(w15) + w10 + SIGMA0_256(w2) + w1;
+	SHA256ROUND(h, a, b, c, d, e, f, g, 17, w1);
+	w2 = SIGMA1_256(w0) + w11 + SIGMA0_256(w3) + w2;
+	SHA256ROUND(g, h, a, b, c, d, e, f, 18, w2);
+	w3 = SIGMA1_256(w1) + w12 + SIGMA0_256(w4) + w3;
+	SHA256ROUND(f, g, h, a, b, c, d, e, 19, w3);
+	w4 = SIGMA1_256(w2) + w13 + SIGMA0_256(w5) + w4;
+	SHA256ROUND(e, f, g, h, a, b, c, d, 20, w4);
+	w5 = SIGMA1_256(w3) + w14 + SIGMA0_256(w6) + w5;
+	SHA256ROUND(d, e, f, g, h, a, b, c, 21, w5);
+	w6 = SIGMA1_256(w4) + w15 + SIGMA0_256(w7) + w6;
+	SHA256ROUND(c, d, e, f, g, h, a, b, 22, w6);
+	w7 = SIGMA1_256(w5) + w0 + SIGMA0_256(w8) + w7;
+	SHA256ROUND(b, c, d, e, f, g, h, a, 23, w7);
+	w8 = SIGMA1_256(w6) + w1 + SIGMA0_256(w9) + w8;
+	SHA256ROUND(a, b, c, d, e, f, g, h, 24, w8);
+	w9 = SIGMA1_256(w7) + w2 + SIGMA0_256(w10) + w9;
+	SHA256ROUND(h, a, b, c, d, e, f, g, 25, w9);
+	w10 = SIGMA1_256(w8) + w3 + SIGMA0_256(w11) + w10;
+	SHA256ROUND(g, h, a, b, c, d, e, f, 26, w10);
+	w11 = SIGMA1_256(w9) + w4 + SIGMA0_256(w12) + w11;
+	SHA256ROUND(f, g, h, a, b, c, d, e, 27, w11);
+	w12 = SIGMA1_256(w10) + w5 + SIGMA0_256(w13) + w12;
+	SHA256ROUND(e, f, g, h, a, b, c, d, 28, w12);
+	w13 = SIGMA1_256(w11) + w6 + SIGMA0_256(w14) + w13;
+	SHA256ROUND(d, e, f, g, h, a, b, c, 29, w13);
+	w14 = SIGMA1_256(w12) + w7 + SIGMA0_256(w15) + w14;
+	SHA256ROUND(c, d, e, f, g, h, a, b, 30, w14);
+	w15 = SIGMA1_256(w13) + w8 + SIGMA0_256(w0) + w15;
+	SHA256ROUND(b, c, d, e, f, g, h, a, 31, w15);
+
+	w0 = SIGMA1_256(w14) + w9 + SIGMA0_256(w1) + w0;
+	SHA256ROUND(a, b, c, d, e, f, g, h, 32, w0);
+	w1 = SIGMA1_256(w15) + w10 + SIGMA0_256(w2) + w1;
+	SHA256ROUND(h, a, b, c, d, e, f, g, 33, w1);
+	w2 = SIGMA1_256(w0) + w11 + SIGMA0_256(w3) + w2;
+	SHA256ROUND(g, h, a, b, c, d, e, f, 34, w2);
+	w3 = SIGMA1_256(w1) + w12 + SIGMA0_256(w4) + w3;
+	SHA256ROUND(f, g, h, a, b, c, d, e, 35, w3);
+	w4 = SIGMA1_256(w2) + w13 + SIGMA0_256(w5) + w4;
+	SHA256ROUND(e, f, g, h, a, b, c, d, 36, w4);
+	w5 = SIGMA1_256(w3) + w14 + SIGMA0_256(w6) + w5;
+	SHA256ROUND(d, e, f, g, h, a, b, c, 37, w5);
+	w6 = SIGMA1_256(w4) + w15 + SIGMA0_256(w7) + w6;
+	SHA256ROUND(c, d, e, f, g, h, a, b, 38, w6);
+	w7 = SIGMA1_256(w5) + w0 + SIGMA0_256(w8) + w7;
+	SHA256ROUND(b, c, d, e, f, g, h, a, 39, w7);
+	w8 = SIGMA1_256(w6) + w1 + SIGMA0_256(w9) + w8;
+	SHA256ROUND(a, b, c, d, e, f, g, h, 40, w8);
+	w9 = SIGMA1_256(w7) + w2 + SIGMA0_256(w10) + w9;
+	SHA256ROUND(h, a, b, c, d, e, f, g, 41, w9);
+	w10 = SIGMA1_256(w8) + w3 + SIGMA0_256(w11) + w10;
+	SHA256ROUND(g, h, a, b, c, d, e, f, 42, w10);
+	w11 = SIGMA1_256(w9) + w4 + SIGMA0_256(w12) + w11;
+	SHA256ROUND(f, g, h, a, b, c, d, e, 43, w11);
+	w12 = SIGMA1_256(w10) + w5 + SIGMA0_256(w13) + w12;
+	SHA256ROUND(e, f, g, h, a, b, c, d, 44, w12);
+	w13 = SIGMA1_256(w11) + w6 + SIGMA0_256(w14) + w13;
+	SHA256ROUND(d, e, f, g, h, a, b, c, 45, w13);
+	w14 = SIGMA1_256(w12) + w7 + SIGMA0_256(w15) + w14;
+	SHA256ROUND(c, d, e, f, g, h, a, b, 46, w14);
+	w15 = SIGMA1_256(w13) + w8 + SIGMA0_256(w0) + w15;
+	SHA256ROUND(b, c, d, e, f, g, h, a, 47, w15);
+
+	w0 = SIGMA1_256(w14) + w9 + SIGMA0_256(w1) + w0;
+	SHA256ROUND(a, b, c, d, e, f, g, h, 48, w0);
+	w1 = SIGMA1_256(w15) + w10 + SIGMA0_256(w2) + w1;
+	SHA256ROUND(h, a, b, c, d, e, f, g, 49, w1);
+	w2 = SIGMA1_256(w0) + w11 + SIGMA0_256(w3) + w2;
+	SHA256ROUND(g, h, a, b, c, d, e, f, 50, w2);
+	w3 = SIGMA1_256(w1) + w12 + SIGMA0_256(w4) + w3;
+	SHA256ROUND(f, g, h, a, b, c, d, e, 51, w3);
+	w4 = SIGMA1_256(w2) + w13 + SIGMA0_256(w5) + w4;
+	SHA256ROUND(e, f, g, h, a, b, c, d, 52, w4);
+	w5 = SIGMA1_256(w3) + w14 + SIGMA0_256(w6) + w5;
+	SHA256ROUND(d, e, f, g, h, a, b, c, 53, w5);
+	w6 = SIGMA1_256(w4) + w15 + SIGMA0_256(w7) + w6;
+	SHA256ROUND(c, d, e, f, g, h, a, b, 54, w6);
+	w7 = SIGMA1_256(w5) + w0 + SIGMA0_256(w8) + w7;
+	SHA256ROUND(b, c, d, e, f, g, h, a, 55, w7);
+	w8 = SIGMA1_256(w6) + w1 + SIGMA0_256(w9) + w8;
+	SHA256ROUND(a, b, c, d, e, f, g, h, 56, w8);
+	w9 = SIGMA1_256(w7) + w2 + SIGMA0_256(w10) + w9;
+	SHA256ROUND(h, a, b, c, d, e, f, g, 57, w9);
+	w10 = SIGMA1_256(w8) + w3 + SIGMA0_256(w11) + w10;
+	SHA256ROUND(g, h, a, b, c, d, e, f, 58, w10);
+	w11 = SIGMA1_256(w9) + w4 + SIGMA0_256(w12) + w11;
+	SHA256ROUND(f, g, h, a, b, c, d, e, 59, w11);
+	w12 = SIGMA1_256(w10) + w5 + SIGMA0_256(w13) + w12;
+	SHA256ROUND(e, f, g, h, a, b, c, d, 60, w12);
+	w13 = SIGMA1_256(w11) + w6 + SIGMA0_256(w14) + w13;
+	SHA256ROUND(d, e, f, g, h, a, b, c, 61, w13);
+	w14 = SIGMA1_256(w12) + w7 + SIGMA0_256(w15) + w14;
+	SHA256ROUND(c, d, e, f, g, h, a, b, 62, w14);
+	w15 = SIGMA1_256(w13) + w8 + SIGMA0_256(w0) + w15;
+	SHA256ROUND(b, c, d, e, f, g, h, a, 63, w15);
+
+	ctx->state.s32[0] += a;
+	ctx->state.s32[1] += b;
+	ctx->state.s32[2] += c;
+	ctx->state.s32[3] += d;
+	ctx->state.s32[4] += e;
+	ctx->state.s32[5] += f;
+	ctx->state.s32[6] += g;
+	ctx->state.s32[7] += h;
+}
+#endif	/* !__amd64 */
+
+
+/*
+ * Encode()
+ *
+ * purpose: to convert a list of numbers from little endian to big endian
+ *   input: uint8_t *	: place to store the converted big endian numbers
+ *	    uint32_t *	: place to get numbers to convert from
+ *          size_t	: the length of the input in bytes
+ *  output: void
+ */
+
+static void
+Encode(uint8_t *_RESTRICT_KYWD output, uint32_t *_RESTRICT_KYWD input,
+    size_t len)
+{
+	size_t		i, j;
+
+	for (i = 0, j = 0; j < len; i++, j += 4) {
+		output[j]	= (input[i] >> 24) & 0xff;
+		output[j + 1]	= (input[i] >> 16) & 0xff;
+		output[j + 2]	= (input[i] >>  8) & 0xff;
+		output[j + 3]	= input[i] & 0xff;
+	}
+}
+
+void
+SHA2Init(uint64_t mech, SHA2_CTX *ctx)
+{
+
+	switch (mech) {
+	case SHA256_MECH_INFO_TYPE:
+	case SHA256_HMAC_MECH_INFO_TYPE:
+	case SHA256_HMAC_GEN_MECH_INFO_TYPE:
+		ctx->state.s32[0] = 0x6a09e667U;
+		ctx->state.s32[1] = 0xbb67ae85U;
+		ctx->state.s32[2] = 0x3c6ef372U;
+		ctx->state.s32[3] = 0xa54ff53aU;
+		ctx->state.s32[4] = 0x510e527fU;
+		ctx->state.s32[5] = 0x9b05688cU;
+		ctx->state.s32[6] = 0x1f83d9abU;
+		ctx->state.s32[7] = 0x5be0cd19U;
+		break;
+	default:
+		cmn_err(CE_PANIC,
+		    "sha2_init: failed to find a supported algorithm: 0x%x",
+		    (uint32_t)mech);
+	}
+
+	ctx->algotype = (uint32_t)mech;
+	ctx->count.c64[0] = ctx->count.c64[1] = 0;
+}
+
+void
+SHA256Init(SHA256_CTX *ctx)
+{
+	SHA2Init(SHA256, ctx);
+}
+
+/*
+ * SHA2Update()
+ *
+ * purpose: continues an sha2 digest operation, using the message block
+ *          to update the context.
+ *   input: SHA2_CTX *	: the context to update
+ *          void *	: the message block
+ *          size_t      : the length of the message block, in bytes
+ *  output: void
+ */
+
+void
+SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len)
+{
+	uint32_t	i, buf_index, buf_len, buf_limit;
+	const uint8_t	*input = inptr;
+	uint32_t	algotype = ctx->algotype;
+#if defined(__amd64)
+	uint32_t	block_count;
+#endif	/* !__amd64 */
+
+
+	/* check for noop */
+	if (input_len == 0)
+		return;
+
+	if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
+		buf_limit = 64;
+
+		/* compute number of bytes mod 64 */
+		buf_index = (ctx->count.c32[1] >> 3) & 0x3F;
+
+		/* update number of bits */
+		if ((ctx->count.c32[1] += (input_len << 3)) < (input_len << 3))
+			ctx->count.c32[0]++;
+
+		ctx->count.c32[0] += (input_len >> 29);
+
+	} else {
+		buf_limit = 128;
+
+		/* compute number of bytes mod 128 */
+		buf_index = (ctx->count.c64[1] >> 3) & 0x7F;
+
+		/* update number of bits */
+		if ((ctx->count.c64[1] += (input_len << 3)) < (input_len << 3))
+			ctx->count.c64[0]++;
+
+		ctx->count.c64[0] += (input_len >> 29);
+	}
+
+	buf_len = buf_limit - buf_index;
+
+	/* transform as many times as possible */
+	i = 0;
+	if (input_len >= buf_len) {
+
+		/*
+		 * general optimization:
+		 *
+		 * only do initial bcopy() and SHA2Transform() if
+		 * buf_index != 0.  if buf_index == 0, we're just
+		 * wasting our time doing the bcopy() since there
+		 * wasn't any data left over from a previous call to
+		 * SHA2Update().
+		 */
+		if (buf_index) {
+			bcopy(input, &ctx->buf_un.buf8[buf_index], buf_len);
+			if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE)
+				SHA256Transform(ctx, ctx->buf_un.buf8);
+
+			i = buf_len;
+		}
+
+#if !defined(__amd64)
+		if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
+			for (; i + buf_limit - 1 < input_len; i += buf_limit) {
+				SHA256Transform(ctx, &input[i]);
+			}
+		}
+
+#else
+		if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
+			block_count = (input_len - i) >> 6;
+			if (block_count > 0) {
+				SHA256TransformBlocks(ctx, &input[i],
+				    block_count);
+				i += block_count << 6;
+			}
+		}
+#endif	/* !__amd64 */
+
+		/*
+		 * general optimization:
+		 *
+		 * if i and input_len are the same, return now instead
+		 * of calling bcopy(), since the bcopy() in this case
+		 * will be an expensive noop.
+		 */
+
+		if (input_len == i)
+			return;
+
+		buf_index = 0;
+	}
+
+	/* buffer remaining input */
+	bcopy(&input[i], &ctx->buf_un.buf8[buf_index], input_len - i);
+}
+
+
+/*
+ * SHA2Final()
+ *
+ * purpose: ends an sha2 digest operation, finalizing the message digest and
+ *          zeroing the context.
+ *   input: uchar_t *	: a buffer to store the digest
+ *			: The function actually uses void* because many
+ *			: callers pass things other than uchar_t here.
+ *          SHA2_CTX *  : the context to finalize, save, and zero
+ *  output: void
+ */
+
+void
+SHA2Final(void *digest, SHA2_CTX *ctx)
+{
+	uint8_t		bitcount_be[sizeof (ctx->count.c32)];
+	uint32_t	index;
+	uint32_t	algotype = ctx->algotype;
+
+	if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
+		index  = (ctx->count.c32[1] >> 3) & 0x3f;
+		Encode(bitcount_be, ctx->count.c32, sizeof (bitcount_be));
+		SHA2Update(ctx, PADDING, ((index < 56) ? 56 : 120) - index);
+		SHA2Update(ctx, bitcount_be, sizeof (bitcount_be));
+		Encode(digest, ctx->state.s32, sizeof (ctx->state.s32));
+	}
+
+	/* zeroize sensitive information */
+	bzero(ctx, sizeof (*ctx));
+}