Illumos Crypto Port module added to enable native encryption in zfs

A port of the Illumos Crypto Framework to a Linux kernel module (found
in module/icp). This is needed to do the actual encryption work. We cannot
use the Linux kernel's built in crypto api because it is only exported to
GPL-licensed modules. Having the ICP also means the crypto code can run on
any of the other kernels under OpenZFS. I ended up porting over most of the
internals of the framework, which means that porting over other API calls (if
we need them) should be fairly easy. Specifically, I have ported over the API
functions related to encryption, digests, macs, and crypto templates. The ICP
is able to use assembly-accelerated encryption on amd64 machines and AES-NI
instructions on Intel chips that support it. There are place-holder
directories for similar assembly optimizations for other architectures
(although they have not been written).

Signed-off-by: Tom Caputi <tcaputi@datto.com>
Signed-off-by: Tony Hutter <hutter2@llnl.gov>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Issue #4329
This commit is contained in:
Tom Caputi
2016-05-12 10:51:24 -04:00
committed by Brian Behlendorf
parent be88e733a6
commit 0b04990a5d
90 changed files with 35834 additions and 80 deletions
+307
View File
@@ -0,0 +1,307 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2008 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#ifndef _IA32_SYS_ASM_LINKAGE_H
#define _IA32_SYS_ASM_LINKAGE_H
#include <sys/stack.h>
#include <sys/trap.h>
#ifdef __cplusplus
extern "C" {
#endif
#ifdef _ASM /* The remainder of this file is only for assembly files */
/*
* make annoying differences in assembler syntax go away
*/
/*
* D16 and A16 are used to insert instructions prefixes; the
* macros help the assembler code be slightly more portable.
*/
#if !defined(__GNUC_AS__)
/*
* /usr/ccs/bin/as prefixes are parsed as separate instructions
*/
#define D16 data16;
#define A16 addr16;
/*
* (There are some weird constructs in constant expressions)
*/
#define _CONST(const) [const]
#define _BITNOT(const) -1!_CONST(const)
#define _MUL(a, b) _CONST(a \* b)
#else
/*
* Why not use the 'data16' and 'addr16' prefixes .. well, the
* assembler doesn't quite believe in real mode, and thus argues with
* us about what we're trying to do.
*/
#define D16 .byte 0x66;
#define A16 .byte 0x67;
#define _CONST(const) (const)
#define _BITNOT(const) ~_CONST(const)
#define _MUL(a, b) _CONST(a * b)
#endif
/*
* C pointers are different sizes between i386 and amd64.
* These constants can be used to compute offsets into pointer arrays.
*/
#if defined(__amd64)
#define CLONGSHIFT 3
#define CLONGSIZE 8
#define CLONGMASK 7
#elif defined(__i386)
#define CLONGSHIFT 2
#define CLONGSIZE 4
#define CLONGMASK 3
#endif
/*
* Since we know we're either ILP32 or LP64 ..
*/
#define CPTRSHIFT CLONGSHIFT
#define CPTRSIZE CLONGSIZE
#define CPTRMASK CLONGMASK
#if CPTRSIZE != (1 << CPTRSHIFT) || CLONGSIZE != (1 << CLONGSHIFT)
#error "inconsistent shift constants"
#endif
#if CPTRMASK != (CPTRSIZE - 1) || CLONGMASK != (CLONGSIZE - 1)
#error "inconsistent mask constants"
#endif
#define ASM_ENTRY_ALIGN 16
/*
* SSE register alignment and save areas
*/
#define XMM_SIZE 16
#define XMM_ALIGN 16
#if defined(__amd64)
#define SAVE_XMM_PROLOG(sreg, nreg) \
subq $_CONST(_MUL(XMM_SIZE, nreg)), %rsp; \
movq %rsp, sreg
#define RSTOR_XMM_EPILOG(sreg, nreg) \
addq $_CONST(_MUL(XMM_SIZE, nreg)), %rsp
#elif defined(__i386)
#define SAVE_XMM_PROLOG(sreg, nreg) \
subl $_CONST(_MUL(XMM_SIZE, nreg) + XMM_ALIGN), %esp; \
movl %esp, sreg; \
addl $XMM_ALIGN, sreg; \
andl $_BITNOT(XMM_ALIGN-1), sreg
#define RSTOR_XMM_EPILOG(sreg, nreg) \
addl $_CONST(_MUL(XMM_SIZE, nreg) + XMM_ALIGN), %esp;
#endif /* __i386 */
/*
* profiling causes definitions of the MCOUNT and RTMCOUNT
* particular to the type
*/
#ifdef GPROF
#define MCOUNT(x) \
pushl %ebp; \
movl %esp, %ebp; \
call _mcount; \
popl %ebp
#endif /* GPROF */
#ifdef PROF
#define MCOUNT(x) \
/* CSTYLED */ \
.lcomm .L_/**/x/**/1, 4, 4; \
pushl %ebp; \
movl %esp, %ebp; \
/* CSTYLED */ \
movl $.L_/**/x/**/1, %edx; \
call _mcount; \
popl %ebp
#endif /* PROF */
/*
* if we are not profiling, MCOUNT should be defined to nothing
*/
#if !defined(PROF) && !defined(GPROF)
#define MCOUNT(x)
#endif /* !defined(PROF) && !defined(GPROF) */
#define RTMCOUNT(x) MCOUNT(x)
/*
* Macro to define weak symbol aliases. These are similar to the ANSI-C
* #pragma weak _name = name
* except a compiler can determine type. The assembler must be told. Hence,
* the second parameter must be the type of the symbol (i.e.: function,...)
*/
#define ANSI_PRAGMA_WEAK(sym, stype) \
/* CSTYLED */ \
.weak _/**/sym; \
/* CSTYLED */ \
.type _/**/sym, @stype; \
/* CSTYLED */ \
_/**/sym = sym
/*
* Like ANSI_PRAGMA_WEAK(), but for unrelated names, as in:
* #pragma weak sym1 = sym2
*/
#define ANSI_PRAGMA_WEAK2(sym1, sym2, stype) \
.weak sym1; \
.type sym1, @stype; \
sym1 = sym2
/*
* ENTRY provides the standard procedure entry code and an easy way to
* insert the calls to mcount for profiling. ENTRY_NP is identical, but
* never calls mcount.
*/
#define ENTRY(x) \
.text; \
.align ASM_ENTRY_ALIGN; \
.globl x; \
.type x, @function; \
x: MCOUNT(x)
#define ENTRY_NP(x) \
.text; \
.align ASM_ENTRY_ALIGN; \
.globl x; \
.type x, @function; \
x:
#define RTENTRY(x) \
.text; \
.align ASM_ENTRY_ALIGN; \
.globl x; \
.type x, @function; \
x: RTMCOUNT(x)
/*
* ENTRY2 is identical to ENTRY but provides two labels for the entry point.
*/
#define ENTRY2(x, y) \
.text; \
.align ASM_ENTRY_ALIGN; \
.globl x, y; \
.type x, @function; \
.type y, @function; \
/* CSTYLED */ \
x: ; \
y: MCOUNT(x)
#define ENTRY_NP2(x, y) \
.text; \
.align ASM_ENTRY_ALIGN; \
.globl x, y; \
.type x, @function; \
.type y, @function; \
/* CSTYLED */ \
x: ; \
y:
/*
* ALTENTRY provides for additional entry points.
*/
#define ALTENTRY(x) \
.globl x; \
.type x, @function; \
x:
/*
* DGDEF and DGDEF2 provide global data declarations.
*
* DGDEF provides a word aligned word of storage.
*
* DGDEF2 allocates "sz" bytes of storage with **NO** alignment. This
* implies this macro is best used for byte arrays.
*
* DGDEF3 allocates "sz" bytes of storage with "algn" alignment.
*/
#define DGDEF2(name, sz) \
.data; \
.globl name; \
.type name, @object; \
.size name, sz; \
name:
#define DGDEF3(name, sz, algn) \
.data; \
.align algn; \
.globl name; \
.type name, @object; \
.size name, sz; \
name:
#define DGDEF(name) DGDEF3(name, 4, 4)
/*
* SET_SIZE trails a function and set the size for the ELF symbol table.
*/
#define SET_SIZE(x) \
.size x, [.-x]
/*
* NWORD provides native word value.
*/
#if defined(__amd64)
/*CSTYLED*/
#define NWORD quad
#elif defined(__i386)
#define NWORD long
#endif /* __i386 */
#endif /* _ASM */
#ifdef __cplusplus
}
#endif
#endif /* _IA32_SYS_ASM_LINKAGE_H */
+160
View File
@@ -0,0 +1,160 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2004 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#ifndef _IA32_SYS_STACK_H
#define _IA32_SYS_STACK_H
#if !defined(_ASM)
#include <sys/types.h>
#endif
#ifdef __cplusplus
extern "C" {
#endif
/*
* In the x86 world, a stack frame looks like this:
*
* |--------------------------|
* 4n+8(%ebp) ->| argument word n |
* | ... | (Previous frame)
* 8(%ebp) ->| argument word 0 |
* |--------------------------|--------------------
* 4(%ebp) ->| return address |
* |--------------------------|
* 0(%ebp) ->| previous %ebp (optional) |
* |--------------------------|
* -4(%ebp) ->| unspecified | (Current frame)
* | ... |
* 0(%esp) ->| variable size |
* |--------------------------|
*/
/*
* Stack alignment macros.
*/
#define STACK_ALIGN32 4
#define STACK_ENTRY_ALIGN32 4
#define STACK_BIAS32 0
#define SA32(x) (((x)+(STACK_ALIGN32-1)) & ~(STACK_ALIGN32-1))
#define STACK_RESERVE32 0
#define MINFRAME32 0
#if defined(__amd64)
/*
* In the amd64 world, a stack frame looks like this:
*
* |--------------------------|
* 8n+16(%rbp)->| argument word n |
* | ... | (Previous frame)
* 16(%rbp) ->| argument word 0 |
* |--------------------------|--------------------
* 8(%rbp) ->| return address |
* |--------------------------|
* 0(%rbp) ->| previous %rbp |
* |--------------------------|
* -8(%rbp) ->| unspecified | (Current frame)
* | ... |
* 0(%rsp) ->| variable size |
* |--------------------------|
* -128(%rsp) ->| reserved for function |
* |--------------------------|
*
* The end of the input argument area must be aligned on a 16-byte
* boundary; i.e. (%rsp - 8) % 16 == 0 at function entry.
*
* The 128-byte location beyond %rsp is considered to be reserved for
* functions and is NOT modified by signal handlers. It can be used
* to store temporary data that is not needed across function calls.
*/
/*
* Stack alignment macros.
*/
#define STACK_ALIGN64 16
#define STACK_ENTRY_ALIGN64 8
#define STACK_BIAS64 0
#define SA64(x) (((x)+(STACK_ALIGN64-1)) & ~(STACK_ALIGN64-1))
#define STACK_RESERVE64 128
#define MINFRAME64 0
#define STACK_ALIGN STACK_ALIGN64
#define STACK_ENTRY_ALIGN STACK_ENTRY_ALIGN64
#define STACK_BIAS STACK_BIAS64
#define SA(x) SA64(x)
#define STACK_RESERVE STACK_RESERVE64
#define MINFRAME MINFRAME64
#elif defined(__i386)
#define STACK_ALIGN STACK_ALIGN32
#define STACK_ENTRY_ALIGN STACK_ENTRY_ALIGN32
#define STACK_BIAS STACK_BIAS32
#define SA(x) SA32(x)
#define STACK_RESERVE STACK_RESERVE32
#define MINFRAME MINFRAME32
#endif /* __i386 */
#if defined(_KERNEL) && !defined(_ASM)
#if defined(DEBUG)
#if STACK_ALIGN == 4
#define ASSERT_STACK_ALIGNED() \
{ \
uint32_t __tmp; \
ASSERT((((uintptr_t)&__tmp) & (STACK_ALIGN - 1)) == 0); \
}
#elif (STACK_ALIGN == 16) && (_LONG_DOUBLE_ALIGNMENT == 16)
#define ASSERT_STACK_ALIGNED() \
{ \
long double __tmp; \
ASSERT((((uintptr_t)&__tmp) & (STACK_ALIGN - 1)) == 0); \
}
#endif
#else /* DEBUG */
#define ASSERT_STACK_ALIGNED()
#endif /* DEBUG */
struct regs;
void traceregs(struct regs *);
void traceback(caddr_t);
#endif /* defined(_KERNEL) && !defined(_ASM) */
#define STACK_GROWTH_DOWN /* stacks grow from high to low addresses */
#ifdef __cplusplus
}
#endif
#endif /* _IA32_SYS_STACK_H */
+107
View File
@@ -0,0 +1,107 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
/* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
/* All Rights Reserved */
/*
* Copyright 2006 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#ifndef _IA32_SYS_TRAP_H
#define _IA32_SYS_TRAP_H
#ifdef __cplusplus
extern "C" {
#endif
/*
* Trap type values
*/
#define T_ZERODIV 0x0 /* #de divide by 0 error */
#define T_SGLSTP 0x1 /* #db single step */
#define T_NMIFLT 0x2 /* NMI */
#define T_BPTFLT 0x3 /* #bp breakpoint fault, INT3 insn */
#define T_OVFLW 0x4 /* #of INTO overflow fault */
#define T_BOUNDFLT 0x5 /* #br BOUND insn fault */
#define T_ILLINST 0x6 /* #ud invalid opcode fault */
#define T_NOEXTFLT 0x7 /* #nm device not available: x87 */
#define T_DBLFLT 0x8 /* #df double fault */
#define T_EXTOVRFLT 0x9 /* [not generated: 386 only] */
#define T_TSSFLT 0xa /* #ts invalid TSS fault */
#define T_SEGFLT 0xb /* #np segment not present fault */
#define T_STKFLT 0xc /* #ss stack fault */
#define T_GPFLT 0xd /* #gp general protection fault */
#define T_PGFLT 0xe /* #pf page fault */
#define T_EXTERRFLT 0x10 /* #mf x87 FPU error fault */
#define T_ALIGNMENT 0x11 /* #ac alignment check error */
#define T_MCE 0x12 /* #mc machine check exception */
#define T_SIMDFPE 0x13 /* #xm SSE/SSE exception */
#define T_DBGENTR 0x14 /* debugger entry */
#define T_ENDPERR 0x21 /* emulated extension error flt */
#define T_ENOEXTFLT 0x20 /* emulated ext not present */
#define T_FASTTRAP 0xd2 /* fast system call */
#define T_SYSCALLINT 0x91 /* general system call */
#define T_DTRACE_RET 0x7f /* DTrace pid return */
#define T_INT80 0x80 /* int80 handler for linux emulation */
#define T_SOFTINT 0x50fd /* pseudo softint trap type */
/*
* Pseudo traps.
*/
#define T_INTERRUPT 0x100
#define T_FAULT 0x200
#define T_AST 0x400
#define T_SYSCALL 0x180
/*
* Values of error code on stack in case of page fault
*/
#define PF_ERR_MASK 0x01 /* Mask for error bit */
#define PF_ERR_PAGE 0x00 /* page not present */
#define PF_ERR_PROT 0x01 /* protection error */
#define PF_ERR_WRITE 0x02 /* fault caused by write (else read) */
#define PF_ERR_USER 0x04 /* processor was in user mode */
/* (else supervisor) */
#define PF_ERR_EXEC 0x10 /* attempt to execute a No eXec page (AMD) */
/*
* Definitions for fast system call subfunctions
*/
#define T_FNULL 0 /* Null trap for testing */
#define T_FGETFP 1 /* Get emulated FP context */
#define T_FSETFP 2 /* Set emulated FP context */
#define T_GETHRTIME 3 /* Get high resolution time */
#define T_GETHRVTIME 4 /* Get high resolution virtual time */
#define T_GETHRESTIME 5 /* Get high resolution time */
#define T_GETLGRP 6 /* Get home lgrpid */
#define T_LASTFAST 6 /* Last valid subfunction */
#ifdef __cplusplus
}
#endif
#endif /* _IA32_SYS_TRAP_H */