Add support for selecting encryption backend

- Add two new module parameters to icp (icp_aes_impl, icp_gcm_impl)
  that control the crypto implementation.  At the moment there is a
  choice between generic and aesni (on platforms that support it).
- This enables support for AES-NI and PCLMULQDQ-NI on AMD Family
  15h (bulldozer) and newer CPUs (zen).
- Modify aes_key_t to track what implementation it was generated
  with as key schedules generated with various implementations
  are not necessarily interchangable.

Reviewed by: Gvozden Neskovic <neskovic@gmail.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Tom Caputi <tcaputi@datto.com>
Reviewed-by: Richard Laager <rlaager@wiktel.com>
Signed-off-by: Nathaniel R. Lewis <linux.robotdude@gmail.com>
Closes #7102 
Closes #7103

module/icp/include/aes/aes_impl.h

@@ -106,17 +106,15 @@ typedef union {
 	uint32_t	ks32[((MAX_AES_NR) + 1) * (MAX_AES_NB)];
 } aes_ks_t;
 
-/* aes_key.flags value: */
-#define	INTEL_AES_NI_CAPABLE	0x1	/* AES-NI instructions present */
-
+typedef struct aes_impl_ops aes_impl_ops_t;
 typedef struct aes_key aes_key_t;
 struct aes_key {
 	aes_ks_t	encr_ks;  /* encryption key schedule */
 	aes_ks_t	decr_ks;  /* decryption key schedule */
 #ifdef __amd64
 	long double	align128; /* Align fields above for Intel AES-NI */
-	int		flags;	  /* implementation-dependent flags */
 #endif	/* __amd64 */
+	const aes_impl_ops_t	*ops;	/* ops associated with this schedule */
 	int		nr;	  /* number of rounds (10, 12, or 14) */
 	int		type;	  /* key schedule size (32 or 64 bits) */
 };
@@ -163,6 +161,50 @@ typedef enum aes_mech_type {
 
 #endif /* _AES_IMPL */
 
+/*
+ * Methods used to define aes implementation
+ *
+ * @aes_gen_f Key generation
+ * @aes_enc_f Function encrypts one block
+ * @aes_dec_f Function decrypts one block
+ * @aes_will_work_f Function tests whether method will function
+ */
+typedef void 		(*aes_generate_f)(aes_key_t *, const uint32_t *, int);
+typedef void		(*aes_encrypt_f)(const uint32_t[], int,
+    const uint32_t[4], uint32_t[4]);
+typedef void		(*aes_decrypt_f)(const uint32_t[], int,
+    const uint32_t[4], uint32_t[4]);
+typedef boolean_t	(*aes_will_work_f)(void);
+
+#define	AES_IMPL_NAME_MAX (16)
+
+struct aes_impl_ops {
+	aes_generate_f generate;
+	aes_encrypt_f encrypt;
+	aes_decrypt_f decrypt;
+	aes_will_work_f is_supported;
+	boolean_t needs_byteswap;
+	char name[AES_IMPL_NAME_MAX];
+};
+
+extern const aes_impl_ops_t aes_generic_impl;
+#if defined(__x86_64)
+extern const aes_impl_ops_t aes_x86_64_impl;
+#endif
+#if defined(__x86_64) && defined(HAVE_AES)
+extern const aes_impl_ops_t aes_aesni_impl;
+#endif
+
+/*
+ * Initializes fastest implementation
+ */
+void aes_impl_init(void);
+
+/*
+ * Get selected aes implementation
+ */
+struct aes_impl_ops *aes_impl_get_ops(void);
+
 #ifdef	__cplusplus
 }
 #endif

module/icp/include/modes/gcm_impl.h

@@ -0,0 +1,75 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ */
+
+#ifndef	_GCM_IMPL_H
+#define	_GCM_IMPL_H
+
+/*
+ * GCM function dispatcher.
+ */
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#include <sys/zfs_context.h>
+#include <sys/crypto/common.h>
+
+/*
+ * Methods used to define gcm implementation
+ *
+ * @gcm_mul_f Perform carry-less multiplication
+ * @gcm_will_work_f Function tests whether implementation will function
+ */
+typedef void 		(*gcm_mul_f)(uint64_t *, uint64_t *, uint64_t *);
+typedef boolean_t	(*gcm_will_work_f)(void);
+
+#define	GCM_IMPL_NAME_MAX (16)
+
+typedef struct gcm_impl_ops {
+	gcm_mul_f mul;
+	gcm_will_work_f is_supported;
+	char name[GCM_IMPL_NAME_MAX];
+} gcm_impl_ops_t;
+
+extern const gcm_impl_ops_t gcm_generic_impl;
+#if defined(__x86_64) && defined(HAVE_PCLMULQDQ)
+extern const gcm_impl_ops_t gcm_pclmulqdq_impl;
+#endif
+
+/*
+ * Initializes fastest implementation
+ */
+void gcm_impl_init(void);
+
+/*
+ * Get selected aes implementation
+ */
+struct gcm_impl_ops *gcm_impl_get_ops(void);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _GCM_IMPL_H */