c/mirror_zfs
1
0
Fork 0
mirror of https://git.proxmox.com/git/mirror_zfs.git synced 2024-12-27 19:39:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

Allow QAT to handle non page-aligned buffers

Browse Source 
This patch adds some handling to the QAT acceleration functions
that allows them to handle buffers that are not aligned with the
page cache. At the moment this never happens since callers only
happen to work with page-aligned buffers, but this code should
prevent headaches if this isn't always true in the future. This
patch also adds some cleanups to align the QAT compression code
with the encryption and checksumming code.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Weigang Li <weigang.li@intel.com>
Signed-off-by: Tom Caputi <tcaputi@datto.com>
Closes #7305
This commit is contained in:
Tom Caputi 2018-03-16 19:02:49 -04:00 committed by Brian Behlendorf
parent cec3a0a1bb
commit 17dd88352e
2 changed files with 60 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
module/zfs/qat_compress.c
View File

@ -241,18 +241,24 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
CpaStatus status = CPA_STATUS_SUCCESS; CpaStatus status = CPA_STATUS_SUCCESS;
Cpa32U hdr_sz = 0; Cpa32U hdr_sz = 0;
Cpa32U compressed_sz; Cpa32U compressed_sz;
Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 1; Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2;
Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 1; Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2;
Cpa32U bytes_left; Cpa32U bytes_left;
char *data; char *data;
struct page *in_page, *out_page; struct page *in_page, *out_page;
struct page **in_pages = NULL; struct page **in_pages = NULL;
struct page **out_pages = NULL; struct page **out_pages = NULL;
Cpa32U page_off = 0;
struct completion complete; struct completion complete;
size_t ret = -1; size_t ret = -1;
Cpa16U page_num = 0; Cpa32U page_num = 0;
Cpa16U i; Cpa16U i;
/*
* We increment num_src_buf and num_dst_buf by 2 to allow
* us to handle non page-aligned buffer addresses and buffers
* whose sizes are not divisible by PAGE_SIZE.
*/
Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) + Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) +
(num_src_buf * sizeof (CpaFlatBuffer)); (num_src_buf * sizeof (CpaFlatBuffer));
Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) + Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) +
@ -306,11 +312,12 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
data = src; data = src;
page_num = 0; page_num = 0;
while (bytes_left > 0) { while (bytes_left > 0) {
page_off = ((long)data & ~PAGE_MASK);
in_page = qat_mem_to_page(data); in_page = qat_mem_to_page(data);
in_pages[page_num] = in_page; in_pages[page_num] = in_page;
flat_buf_src->pData = kmap(in_page); flat_buf_src->pData = kmap(in_page) + page_off;
flat_buf_src->dataLenInBytes = flat_buf_src->dataLenInBytes =
min((long)bytes_left, (long)PAGE_SIZE); min((long)PAGE_SIZE - page_off, (long)bytes_left);
bytes_left -= flat_buf_src->dataLenInBytes; bytes_left -= flat_buf_src->dataLenInBytes;
data += flat_buf_src->dataLenInBytes; data += flat_buf_src->dataLenInBytes;
@ -325,11 +332,12 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
data = dst; data = dst;
page_num = 0; page_num = 0;
while (bytes_left > 0) { while (bytes_left > 0) {
page_off = ((long)data & ~PAGE_MASK);
out_page = qat_mem_to_page(data); out_page = qat_mem_to_page(data);
flat_buf_dst->pData = kmap(out_page); flat_buf_dst->pData = kmap(out_page) + page_off;
out_pages[page_num] = out_page; out_pages[page_num] = out_page;
flat_buf_dst->dataLenInBytes = flat_buf_dst->dataLenInBytes =
min((long)bytes_left, (long)PAGE_SIZE); min((long)PAGE_SIZE - page_off, (long)bytes_left);
bytes_left -= flat_buf_dst->dataLenInBytes; bytes_left -= flat_buf_dst->dataLenInBytes;
data += flat_buf_dst->dataLenInBytes; data += flat_buf_dst->dataLenInBytes;

72
module/zfs/qat_crypt.c
View File

@ -266,13 +266,11 @@ qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
if (status != CPA_STATUS_SUCCESS) if (status != CPA_STATUS_SUCCESS)
return (status); return (status);
src->numBuffers = nr_bufs;
status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size); status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
if (status != CPA_STATUS_SUCCESS) if (status != CPA_STATUS_SUCCESS)
goto error; goto error;
if (src != dst) { if (src != dst) {
dst->numBuffers = nr_bufs;
status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData, status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
meta_size); meta_size);
if (status != CPA_STATUS_SUCCESS) if (status != CPA_STATUS_SUCCESS)
@ -297,10 +295,9 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
CpaStatus status = CPA_STATUS_SUCCESS; CpaStatus status = CPA_STATUS_SUCCESS;
Cpa16U i; Cpa16U i;
CpaInstanceHandle cy_inst_handle; CpaInstanceHandle cy_inst_handle;
Cpa16U nr_bufs = (enc_len + PAGE_CACHE_SIZE - 1) / PAGE_CACHE_SIZE; Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
Cpa32U bytes_left = 0; Cpa32U bytes_left = 0;
Cpa8S *in = NULL; Cpa8S *data = NULL;
Cpa8S *out = NULL;
CpaCySymSessionCtx *cy_session_ctx = NULL; CpaCySymSessionCtx *cy_session_ctx = NULL;
cy_callback_t cb; cy_callback_t cb;
CpaCySymOpData op_data = { 0 }; CpaCySymOpData op_data = { 0 };
@ -312,7 +309,8 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
CpaFlatBuffer *flat_dst_buf = NULL; CpaFlatBuffer *flat_dst_buf = NULL;
struct page *in_pages[MAX_PAGE_NUM]; struct page *in_pages[MAX_PAGE_NUM];
struct page *out_pages[MAX_PAGE_NUM]; struct page *out_pages[MAX_PAGE_NUM];
Cpa32S page_num = 0; Cpa32U in_page_num = 0;
Cpa32U out_page_num = 0;
Cpa32U in_page_off = 0; Cpa32U in_page_off = 0;
Cpa32U out_page_off = 0; Cpa32U out_page_off = 0;
@ -336,6 +334,11 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
return (status); return (status);
} }
/*
* We increment nr_bufs by 2 to allow us to handle non
* page-aligned buffer addresses and buffers whose sizes
* are not divisible by PAGE_SIZE.
*/
status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs, status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
&src_buffer_list, &dst_buffer_list); &src_buffer_list, &dst_buffer_list);
if (status != CPA_STATUS_SUCCESS) if (status != CPA_STATUS_SUCCESS)
@ -351,30 +354,39 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
goto fail; goto fail;
bytes_left = enc_len; bytes_left = enc_len;
in = src_buf; data = src_buf;
out = dst_buf;
flat_src_buf = flat_src_buf_array; flat_src_buf = flat_src_buf_array;
flat_dst_buf = flat_dst_buf_array;
while (bytes_left > 0) { while (bytes_left > 0) {
in_page_off = ((long)in & ~PAGE_MASK); in_page_off = ((long)data & ~PAGE_MASK);
out_page_off = ((long)out & ~PAGE_MASK); in_pages[in_page_num] = qat_mem_to_page(data);
in_pages[page_num] = qat_mem_to_page(in); flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
out_pages[page_num] = qat_mem_to_page(out);
flat_src_buf->pData = kmap(in_pages[page_num]) + in_page_off;
flat_dst_buf->pData = kmap(out_pages[page_num]) + out_page_off;
flat_src_buf->dataLenInBytes = flat_src_buf->dataLenInBytes =
min((long)PAGE_CACHE_SIZE - in_page_off, (long)bytes_left); min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
flat_dst_buf->dataLenInBytes = data += flat_src_buf->dataLenInBytes;
min((long)PAGE_CACHE_SIZE - out_page_off, (long)bytes_left);
in += flat_src_buf->dataLenInBytes;
out += flat_dst_buf->dataLenInBytes;
bytes_left -= flat_src_buf->dataLenInBytes; bytes_left -= flat_src_buf->dataLenInBytes;
flat_src_buf++; flat_src_buf++;
flat_dst_buf++; in_page_num++;
page_num++;
} }
src_buffer_list.pBuffers = flat_src_buf_array; src_buffer_list.pBuffers = flat_src_buf_array;
src_buffer_list.numBuffers = in_page_num;
bytes_left = enc_len;
data = dst_buf;
flat_dst_buf = flat_dst_buf_array;
while (bytes_left > 0) {
out_page_off = ((long)data & ~PAGE_MASK);
out_pages[out_page_num] = qat_mem_to_page(data);
flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
out_page_off;
flat_dst_buf->dataLenInBytes =
min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
data += flat_dst_buf->dataLenInBytes;
bytes_left -= flat_dst_buf->dataLenInBytes;
flat_dst_buf++;
out_page_num++;
}
dst_buffer_list.pBuffers = flat_dst_buf_array; dst_buffer_list.pBuffers = flat_dst_buf_array;
dst_buffer_list.numBuffers = out_page_num;
op_data.sessionCtx = cy_session_ctx; op_data.sessionCtx = cy_session_ctx;
op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL; op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
@ -418,10 +430,10 @@ fail:
if (status != CPA_STATUS_SUCCESS) if (status != CPA_STATUS_SUCCESS)
QAT_STAT_BUMP(crypt_fails); QAT_STAT_BUMP(crypt_fails);
for (i = 0; i < page_num; i++) { for (i = 0; i < in_page_num; i++)
kunmap(in_pages[i]); kunmap(in_pages[i]);
for (i = 0; i < out_page_num; i++)
kunmap(out_pages[i]); kunmap(out_pages[i]);
}
cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx); cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData); QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
@ -439,7 +451,7 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
CpaStatus status; CpaStatus status;
Cpa16U i; Cpa16U i;
CpaInstanceHandle cy_inst_handle; CpaInstanceHandle cy_inst_handle;
Cpa16U nr_bufs = (size + PAGE_CACHE_SIZE - 1) / PAGE_CACHE_SIZE; Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
Cpa32U bytes_left = 0; Cpa32U bytes_left = 0;
Cpa8S *data = NULL; Cpa8S *data = NULL;
CpaCySymSessionCtx *cy_session_ctx = NULL; CpaCySymSessionCtx *cy_session_ctx = NULL;
@ -450,7 +462,7 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
CpaFlatBuffer *flat_src_buf_array = NULL; CpaFlatBuffer *flat_src_buf_array = NULL;
CpaFlatBuffer *flat_src_buf = NULL; CpaFlatBuffer *flat_src_buf = NULL;
struct page *in_pages[MAX_PAGE_NUM]; struct page *in_pages[MAX_PAGE_NUM];
Cpa32S page_num = 0; Cpa32U page_num = 0;
Cpa32U page_off = 0; Cpa32U page_off = 0;
QAT_STAT_BUMP(cksum_requests); QAT_STAT_BUMP(cksum_requests);
@ -469,6 +481,11 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
return (status); return (status);
} }
/*
* We increment nr_bufs by 2 to allow us to handle non
* page-aligned buffer addresses and buffers whose sizes
* are not divisible by PAGE_SIZE.
*/
status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs, status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
&src_buffer_list, &src_buffer_list); &src_buffer_list, &src_buffer_list);
if (status != CPA_STATUS_SUCCESS) if (status != CPA_STATUS_SUCCESS)
@ -487,13 +504,14 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
in_pages[page_num] = qat_mem_to_page(data); in_pages[page_num] = qat_mem_to_page(data);
flat_src_buf->pData = kmap(in_pages[page_num]) + page_off; flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
flat_src_buf->dataLenInBytes = flat_src_buf->dataLenInBytes =
min((long)PAGE_CACHE_SIZE - page_off, (long)bytes_left); min((long)PAGE_SIZE - page_off, (long)bytes_left);
data += flat_src_buf->dataLenInBytes; data += flat_src_buf->dataLenInBytes;
bytes_left -= flat_src_buf->dataLenInBytes; bytes_left -= flat_src_buf->dataLenInBytes;
flat_src_buf++; flat_src_buf++;
page_num++; page_num++;
} }
src_buffer_list.pBuffers = flat_src_buf_array; src_buffer_list.pBuffers = flat_src_buf_array;
src_buffer_list.numBuffers = page_num;
op_data.sessionCtx = cy_session_ctx; op_data.sessionCtx = cy_session_ctx;
op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL; op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;