From 34ff636f322c6d5e4f0a52338fa63a7dff09a495 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 18 Jul 2026 13:15:36 -0400 Subject: scftorture: Count single_rpc offline failures in statistics output scf_torture_stats_print() aggregates each invoker thread's counters into a local scf_statistics structure before printing, but the n_single_rpc_ofl field is missing from the aggregation loop. As a result, the "single_rpc_ofl" value printed in the statistics line is always zero, even when smp_call_function_single() invocations for the RPC test have failed due to offline CPUs and been counted by the invoker threads. Add the missing accumulation so that the printed value reflects the actual counts. Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- crypto/async_tx/Kconfig | 28 ++ crypto/async_tx/Makefile | 7 + crypto/async_tx/async_memcpy.c | 97 ++++++ crypto/async_tx/async_pq.c | 464 ++++++++++++++++++++++++++++ crypto/async_tx/async_raid6_recov.c | 595 ++++++++++++++++++++++++++++++++++++ crypto/async_tx/async_tx.c | 281 +++++++++++++++++ crypto/async_tx/async_xor.c | 380 +++++++++++++++++++++++ crypto/async_tx/raid6test.c | 250 +++++++++++++++ 8 files changed, 2102 insertions(+) create mode 100644 crypto/async_tx/Kconfig create mode 100644 crypto/async_tx/Makefile create mode 100644 crypto/async_tx/async_memcpy.c create mode 100644 crypto/async_tx/async_pq.c create mode 100644 crypto/async_tx/async_raid6_recov.c create mode 100644 crypto/async_tx/async_tx.c create mode 100644 crypto/async_tx/async_xor.c create mode 100644 crypto/async_tx/raid6test.c (limited to 'crypto/async_tx') diff --git a/crypto/async_tx/Kconfig b/crypto/async_tx/Kconfig new file mode 100644 index 000000000..89bafa2ef --- /dev/null +++ b/crypto/async_tx/Kconfig @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0 +config ASYNC_CORE + tristate + +config ASYNC_MEMCPY + tristate + select ASYNC_CORE + +config ASYNC_XOR + tristate + select ASYNC_CORE + select XOR_BLOCKS + +config ASYNC_PQ + tristate + select ASYNC_CORE + +config ASYNC_RAID6_RECOV + tristate + select ASYNC_CORE + select ASYNC_PQ + select ASYNC_XOR + +config ASYNC_TX_DISABLE_PQ_VAL_DMA + bool + +config ASYNC_TX_DISABLE_XOR_VAL_DMA + bool diff --git a/crypto/async_tx/Makefile b/crypto/async_tx/Makefile new file mode 100644 index 000000000..056e48245 --- /dev/null +++ b/crypto/async_tx/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_ASYNC_CORE) += async_tx.o +obj-$(CONFIG_ASYNC_MEMCPY) += async_memcpy.o +obj-$(CONFIG_ASYNC_XOR) += async_xor.o +obj-$(CONFIG_ASYNC_PQ) += async_pq.o +obj-$(CONFIG_ASYNC_RAID6_RECOV) += async_raid6_recov.o +obj-$(CONFIG_ASYNC_RAID6_TEST) += raid6test.o diff --git a/crypto/async_tx/async_memcpy.c b/crypto/async_tx/async_memcpy.c new file mode 100644 index 000000000..c538e30e9 --- /dev/null +++ b/crypto/async_tx/async_memcpy.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * copy offload engine support + * + * Copyright © 2006, Intel Corporation. + * + * Dan Williams + * + * with architecture considerations by: + * Neil Brown + * Jeff Garzik + */ +#include +#include +#include +#include +#include +#include + +/** + * async_memcpy - attempt to copy memory with a dma engine. + * @dest: destination page + * @src: src page + * @dest_offset: offset into 'dest' to start transaction + * @src_offset: offset into 'src' to start transaction + * @len: length in bytes + * @submit: submission / completion modifiers + * + * honored flags: ASYNC_TX_ACK + */ +struct dma_async_tx_descriptor * +async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, + unsigned int src_offset, size_t len, + struct async_submit_ctl *submit) +{ + struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMCPY, + &dest, 1, &src, 1, len); + struct dma_device *device = chan ? chan->device : NULL; + struct dma_async_tx_descriptor *tx = NULL; + struct dmaengine_unmap_data *unmap = NULL; + + if (device) + unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT); + + if (unmap && is_dma_copy_aligned(device, src_offset, dest_offset, len)) { + unsigned long dma_prep_flags = 0; + + if (submit->cb_fn) + dma_prep_flags |= DMA_PREP_INTERRUPT; + if (submit->flags & ASYNC_TX_FENCE) + dma_prep_flags |= DMA_PREP_FENCE; + + unmap->to_cnt = 1; + unmap->addr[0] = dma_map_page(device->dev, src, src_offset, len, + DMA_TO_DEVICE); + unmap->from_cnt = 1; + unmap->addr[1] = dma_map_page(device->dev, dest, dest_offset, len, + DMA_FROM_DEVICE); + unmap->len = len; + + tx = device->device_prep_dma_memcpy(chan, unmap->addr[1], + unmap->addr[0], len, + dma_prep_flags); + } + + if (tx) { + pr_debug("%s: (async) len: %zu\n", __func__, len); + + dma_set_unmap(tx, unmap); + async_tx_submit(chan, tx, submit); + } else { + void *dest_buf, *src_buf; + pr_debug("%s: (sync) len: %zu\n", __func__, len); + + /* wait for any prerequisite operations */ + async_tx_quiesce(&submit->depend_tx); + + dest_buf = kmap_atomic(dest) + dest_offset; + src_buf = kmap_atomic(src) + src_offset; + + memcpy(dest_buf, src_buf, len); + + kunmap_atomic(src_buf); + kunmap_atomic(dest_buf); + + async_tx_sync_epilog(submit); + } + + dmaengine_unmap_put(unmap); + + return tx; +} +EXPORT_SYMBOL_GPL(async_memcpy); + +MODULE_AUTHOR("Intel Corporation"); +MODULE_DESCRIPTION("asynchronous memcpy api"); +MODULE_LICENSE("GPL"); diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c new file mode 100644 index 000000000..27f99349e --- /dev/null +++ b/crypto/async_tx/async_pq.c @@ -0,0 +1,464 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright(c) 2007 Yuri Tikhonov + * Copyright(c) 2009 Intel Corporation + */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * struct pq_scribble_page - space to hold throwaway P or Q buffer for + * synchronous gen_syndrome + */ +static struct page *pq_scribble_page; + +/* the struct page *blocks[] parameter passed to async_gen_syndrome() + * and async_syndrome_val() contains the 'P' destination address at + * blocks[disks-2] and the 'Q' destination address at blocks[disks-1] + * + * note: these are macros as they are used as lvalues + */ +#define P(b, d) (b[d-2]) +#define Q(b, d) (b[d-1]) + +#define MAX_DISKS 255 + +/* + * do_async_gen_syndrome - asynchronously calculate P and/or Q + */ +static __async_inline struct dma_async_tx_descriptor * +do_async_gen_syndrome(struct dma_chan *chan, + const unsigned char *scfs, int disks, + struct dmaengine_unmap_data *unmap, + enum dma_ctrl_flags dma_flags, + struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct dma_device *dma = chan->device; + enum async_tx_flags flags_orig = submit->flags; + dma_async_tx_callback cb_fn_orig = submit->cb_fn; + dma_async_tx_callback cb_param_orig = submit->cb_param; + int src_cnt = disks - 2; + unsigned short pq_src_cnt; + dma_addr_t dma_dest[2]; + int src_off = 0; + + while (src_cnt > 0) { + submit->flags = flags_orig; + pq_src_cnt = min(src_cnt, dma_maxpq(dma, dma_flags)); + /* if we are submitting additional pqs, leave the chain open, + * clear the callback parameters, and leave the destination + * buffers mapped + */ + if (src_cnt > pq_src_cnt) { + submit->flags &= ~ASYNC_TX_ACK; + submit->flags |= ASYNC_TX_FENCE; + submit->cb_fn = NULL; + submit->cb_param = NULL; + } else { + submit->cb_fn = cb_fn_orig; + submit->cb_param = cb_param_orig; + if (cb_fn_orig) + dma_flags |= DMA_PREP_INTERRUPT; + } + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; + + /* Drivers force forward progress in case they can not provide + * a descriptor + */ + for (;;) { + dma_dest[0] = unmap->addr[disks - 2]; + dma_dest[1] = unmap->addr[disks - 1]; + tx = dma->device_prep_dma_pq(chan, dma_dest, + &unmap->addr[src_off], + pq_src_cnt, + &scfs[src_off], unmap->len, + dma_flags); + if (likely(tx)) + break; + async_tx_quiesce(&submit->depend_tx); + dma_async_issue_pending(chan); + } + + dma_set_unmap(tx, unmap); + async_tx_submit(chan, tx, submit); + submit->depend_tx = tx; + + /* drop completed sources */ + src_cnt -= pq_src_cnt; + src_off += pq_src_cnt; + + dma_flags |= DMA_PREP_CONTINUE; + } + + return tx; +} + +/* + * do_sync_gen_syndrome - synchronously calculate a raid6 syndrome + */ +static void +do_sync_gen_syndrome(struct page **blocks, unsigned int *offsets, int disks, + size_t len, struct async_submit_ctl *submit) +{ + void **srcs; + int i; + int start = -1, stop = disks - 3; + + if (submit->scribble) + srcs = submit->scribble; + else + srcs = (void **) blocks; + + for (i = 0; i < disks; i++) { + if (blocks[i] == NULL) { + BUG_ON(i > disks - 3); /* P or Q can't be zero */ + srcs[i] = page_address(ZERO_PAGE(0)); + } else { + srcs[i] = page_address(blocks[i]) + offsets[i]; + + if (i < disks - 2) { + stop = i; + if (start == -1) + start = i; + } + } + } + if (submit->flags & ASYNC_TX_PQ_XOR_DST) { + BUG_ON(!raid6_can_xor_syndrome()); + if (start >= 0) + raid6_xor_syndrome(disks, start, stop, len, srcs); + } else + raid6_gen_syndrome(disks, len, srcs); + async_tx_sync_epilog(submit); +} + +static inline bool +is_dma_pq_aligned_offs(struct dma_device *dev, unsigned int *offs, + int src_cnt, size_t len) +{ + int i; + + for (i = 0; i < src_cnt; i++) { + if (!is_dma_pq_aligned(dev, offs[i], 0, len)) + return false; + } + return true; +} + +/** + * async_gen_syndrome - asynchronously calculate a raid6 syndrome + * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1 + * @offsets: offset array into each block (src and dest) to start transaction + * @disks: number of blocks (including missing P or Q, see below) + * @len: length of operation in bytes + * @submit: submission/completion modifiers + * + * General note: This routine assumes a field of GF(2^8) with a + * primitive polynomial of 0x11d and a generator of {02}. + * + * 'disks' note: callers can optionally omit either P or Q (but not + * both) from the calculation by setting blocks[disks-2] or + * blocks[disks-1] to NULL. When P or Q is omitted 'len' must be <= + * PAGE_SIZE as a temporary buffer of this size is used in the + * synchronous path. 'disks' always accounts for both destination + * buffers. If any source buffers (blocks[i] where i < disks - 2) are + * set to NULL those buffers will be replaced with the raid6_zero_page + * in the synchronous path and omitted in the hardware-asynchronous + * path. + */ +struct dma_async_tx_descriptor * +async_gen_syndrome(struct page **blocks, unsigned int *offsets, int disks, + size_t len, struct async_submit_ctl *submit) +{ + int src_cnt = disks - 2; + struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, + &P(blocks, disks), 2, + blocks, src_cnt, len); + struct dma_device *device = chan ? chan->device : NULL; + struct dmaengine_unmap_data *unmap = NULL; + + BUG_ON(disks > MAX_DISKS || !(P(blocks, disks) || Q(blocks, disks))); + + if (device) + unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOWAIT); + + /* XORing P/Q is only implemented in software */ + if (unmap && !(submit->flags & ASYNC_TX_PQ_XOR_DST) && + (src_cnt <= dma_maxpq(device, 0) || + dma_maxpq(device, DMA_PREP_CONTINUE) > 0) && + is_dma_pq_aligned_offs(device, offsets, disks, len)) { + struct dma_async_tx_descriptor *tx; + enum dma_ctrl_flags dma_flags = 0; + unsigned char coefs[MAX_DISKS]; + int i, j; + + /* run the p+q asynchronously */ + pr_debug("%s: (async) disks: %d len: %zu\n", + __func__, disks, len); + + /* convert source addresses being careful to collapse 'empty' + * sources and update the coefficients accordingly + */ + unmap->len = len; + for (i = 0, j = 0; i < src_cnt; i++) { + if (blocks[i] == NULL) + continue; + unmap->addr[j] = dma_map_page(device->dev, blocks[i], + offsets[i], len, DMA_TO_DEVICE); + coefs[j] = raid6_gfexp[i]; + unmap->to_cnt++; + j++; + } + + /* + * DMAs use destinations as sources, + * so use BIDIRECTIONAL mapping + */ + unmap->bidi_cnt++; + if (P(blocks, disks)) + unmap->addr[j++] = dma_map_page(device->dev, P(blocks, disks), + P(offsets, disks), + len, DMA_BIDIRECTIONAL); + else { + unmap->addr[j++] = 0; + dma_flags |= DMA_PREP_PQ_DISABLE_P; + } + + unmap->bidi_cnt++; + if (Q(blocks, disks)) + unmap->addr[j++] = dma_map_page(device->dev, Q(blocks, disks), + Q(offsets, disks), + len, DMA_BIDIRECTIONAL); + else { + unmap->addr[j++] = 0; + dma_flags |= DMA_PREP_PQ_DISABLE_Q; + } + + tx = do_async_gen_syndrome(chan, coefs, j, unmap, dma_flags, submit); + dmaengine_unmap_put(unmap); + return tx; + } + + dmaengine_unmap_put(unmap); + + /* run the pq synchronously */ + pr_debug("%s: (sync) disks: %d len: %zu\n", __func__, disks, len); + + /* wait for any prerequisite operations */ + async_tx_quiesce(&submit->depend_tx); + + if (!P(blocks, disks)) { + P(blocks, disks) = pq_scribble_page; + P(offsets, disks) = 0; + } + if (!Q(blocks, disks)) { + Q(blocks, disks) = pq_scribble_page; + Q(offsets, disks) = 0; + } + do_sync_gen_syndrome(blocks, offsets, disks, len, submit); + + return NULL; +} +EXPORT_SYMBOL_GPL(async_gen_syndrome); + +static inline struct dma_chan * +pq_val_chan(struct async_submit_ctl *submit, struct page **blocks, int disks, size_t len) +{ + #ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA + return NULL; + #endif + return async_tx_find_channel(submit, DMA_PQ_VAL, NULL, 0, blocks, + disks, len); +} + +/** + * async_syndrome_val - asynchronously validate a raid6 syndrome + * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1 + * @offsets: common offset into each block (src and dest) to start transaction + * @disks: number of blocks (including missing P or Q, see below) + * @len: length of operation in bytes + * @pqres: on val failure SUM_CHECK_P_RESULT and/or SUM_CHECK_Q_RESULT are set + * @spare: temporary result buffer for the synchronous case + * @s_off: spare buffer page offset + * @submit: submission / completion modifiers + * + * The same notes from async_gen_syndrome apply to the 'blocks', + * and 'disks' parameters of this routine. The synchronous path + * requires a temporary result buffer and submit->scribble to be + * specified. + */ +struct dma_async_tx_descriptor * +async_syndrome_val(struct page **blocks, unsigned int *offsets, int disks, + size_t len, enum sum_check_flags *pqres, struct page *spare, + unsigned int s_off, struct async_submit_ctl *submit) +{ + struct dma_chan *chan = pq_val_chan(submit, blocks, disks, len); + struct dma_device *device = chan ? chan->device : NULL; + struct dma_async_tx_descriptor *tx; + unsigned char coefs[MAX_DISKS]; + enum dma_ctrl_flags dma_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0; + struct dmaengine_unmap_data *unmap = NULL; + + BUG_ON(disks < 4 || disks > MAX_DISKS); + + if (device) + unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOWAIT); + + if (unmap && disks <= dma_maxpq(device, 0) && + is_dma_pq_aligned_offs(device, offsets, disks, len)) { + struct device *dev = device->dev; + dma_addr_t pq[2]; + int i, j = 0, src_cnt = 0; + + pr_debug("%s: (async) disks: %d len: %zu\n", + __func__, disks, len); + + unmap->len = len; + for (i = 0; i < disks-2; i++) + if (likely(blocks[i])) { + unmap->addr[j] = dma_map_page(dev, blocks[i], + offsets[i], len, + DMA_TO_DEVICE); + coefs[j] = raid6_gfexp[i]; + unmap->to_cnt++; + src_cnt++; + j++; + } + + if (!P(blocks, disks)) { + pq[0] = 0; + dma_flags |= DMA_PREP_PQ_DISABLE_P; + } else { + pq[0] = dma_map_page(dev, P(blocks, disks), + P(offsets, disks), len, + DMA_TO_DEVICE); + unmap->addr[j++] = pq[0]; + unmap->to_cnt++; + } + if (!Q(blocks, disks)) { + pq[1] = 0; + dma_flags |= DMA_PREP_PQ_DISABLE_Q; + } else { + pq[1] = dma_map_page(dev, Q(blocks, disks), + Q(offsets, disks), len, + DMA_TO_DEVICE); + unmap->addr[j++] = pq[1]; + unmap->to_cnt++; + } + + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; + for (;;) { + tx = device->device_prep_dma_pq_val(chan, pq, + unmap->addr, + src_cnt, + coefs, + len, pqres, + dma_flags); + if (likely(tx)) + break; + async_tx_quiesce(&submit->depend_tx); + dma_async_issue_pending(chan); + } + + dma_set_unmap(tx, unmap); + async_tx_submit(chan, tx, submit); + } else { + struct page *p_src = P(blocks, disks); + unsigned int p_off = P(offsets, disks); + struct page *q_src = Q(blocks, disks); + unsigned int q_off = Q(offsets, disks); + enum async_tx_flags flags_orig = submit->flags; + dma_async_tx_callback cb_fn_orig = submit->cb_fn; + void *scribble = submit->scribble; + void *cb_param_orig = submit->cb_param; + void *p, *q, *s; + + pr_debug("%s: (sync) disks: %d len: %zu\n", + __func__, disks, len); + + /* caller must provide a temporary result buffer and + * allow the input parameters to be preserved + */ + BUG_ON(!spare || !scribble); + + /* wait for any prerequisite operations */ + async_tx_quiesce(&submit->depend_tx); + + /* recompute p and/or q into the temporary buffer and then + * check to see the result matches the current value + */ + tx = NULL; + *pqres = 0; + if (p_src) { + init_async_submit(submit, ASYNC_TX_XOR_ZERO_DST, NULL, + NULL, NULL, scribble); + tx = async_xor_offs(spare, s_off, + blocks, offsets, disks-2, len, submit); + async_tx_quiesce(&tx); + p = page_address(p_src) + p_off; + s = page_address(spare) + s_off; + *pqres |= !!memcmp(p, s, len) << SUM_CHECK_P; + } + + if (q_src) { + P(blocks, disks) = NULL; + Q(blocks, disks) = spare; + Q(offsets, disks) = s_off; + init_async_submit(submit, 0, NULL, NULL, NULL, scribble); + tx = async_gen_syndrome(blocks, offsets, disks, + len, submit); + async_tx_quiesce(&tx); + q = page_address(q_src) + q_off; + s = page_address(spare) + s_off; + *pqres |= !!memcmp(q, s, len) << SUM_CHECK_Q; + } + + /* restore P, Q and submit */ + P(blocks, disks) = p_src; + P(offsets, disks) = p_off; + Q(blocks, disks) = q_src; + Q(offsets, disks) = q_off; + + submit->cb_fn = cb_fn_orig; + submit->cb_param = cb_param_orig; + submit->flags = flags_orig; + async_tx_sync_epilog(submit); + tx = NULL; + } + dmaengine_unmap_put(unmap); + + return tx; +} +EXPORT_SYMBOL_GPL(async_syndrome_val); + +static int __init async_pq_init(void) +{ + pq_scribble_page = alloc_page(GFP_KERNEL); + + if (pq_scribble_page) + return 0; + + pr_err("%s: failed to allocate required spare page\n", __func__); + + return -ENOMEM; +} + +static void __exit async_pq_exit(void) +{ + __free_page(pq_scribble_page); +} + +module_init(async_pq_init); +module_exit(async_pq_exit); + +MODULE_DESCRIPTION("asynchronous raid6 syndrome generation/validation"); +MODULE_LICENSE("GPL"); diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c new file mode 100644 index 000000000..e53870d84 --- /dev/null +++ b/crypto/async_tx/async_raid6_recov.c @@ -0,0 +1,595 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Asynchronous RAID-6 recovery calculations ASYNC_TX API. + * Copyright(c) 2009 Intel Corporation + * + * based on raid6recov.c: + * Copyright 2002 H. Peter Anvin + */ +#include +#include +#include +#include +#include +#include +#include +#include + +static struct dma_async_tx_descriptor * +async_sum_product(struct page *dest, unsigned int d_off, + struct page **srcs, unsigned int *src_offs, unsigned char *coef, + size_t len, struct async_submit_ctl *submit) +{ + struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, + &dest, 1, srcs, 2, len); + struct dma_device *dma = chan ? chan->device : NULL; + struct dmaengine_unmap_data *unmap = NULL; + const u8 *amul, *bmul; + u8 ax, bx; + u8 *a, *b, *c; + + if (dma) + unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT); + + if (unmap) { + struct device *dev = dma->dev; + dma_addr_t pq[2]; + struct dma_async_tx_descriptor *tx; + enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P; + + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; + unmap->addr[0] = dma_map_page(dev, srcs[0], src_offs[0], + len, DMA_TO_DEVICE); + unmap->addr[1] = dma_map_page(dev, srcs[1], src_offs[1], + len, DMA_TO_DEVICE); + unmap->to_cnt = 2; + + unmap->addr[2] = dma_map_page(dev, dest, d_off, + len, DMA_BIDIRECTIONAL); + unmap->bidi_cnt = 1; + /* engine only looks at Q, but expects it to follow P */ + pq[1] = unmap->addr[2]; + + unmap->len = len; + tx = dma->device_prep_dma_pq(chan, pq, unmap->addr, 2, coef, + len, dma_flags); + if (tx) { + dma_set_unmap(tx, unmap); + async_tx_submit(chan, tx, submit); + dmaengine_unmap_put(unmap); + return tx; + } + + /* could not get a descriptor, unmap and fall through to + * the synchronous path + */ + dmaengine_unmap_put(unmap); + } + + /* run the operation synchronously */ + async_tx_quiesce(&submit->depend_tx); + amul = raid6_gfmul[coef[0]]; + bmul = raid6_gfmul[coef[1]]; + a = page_address(srcs[0]) + src_offs[0]; + b = page_address(srcs[1]) + src_offs[1]; + c = page_address(dest) + d_off; + + while (len--) { + ax = amul[*a++]; + bx = bmul[*b++]; + *c++ = ax ^ bx; + } + + return NULL; +} + +static struct dma_async_tx_descriptor * +async_mult(struct page *dest, unsigned int d_off, struct page *src, + unsigned int s_off, u8 coef, size_t len, + struct async_submit_ctl *submit) +{ + struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, + &dest, 1, &src, 1, len); + struct dma_device *dma = chan ? chan->device : NULL; + struct dmaengine_unmap_data *unmap = NULL; + const u8 *qmul; /* Q multiplier table */ + u8 *d, *s; + + if (dma) + unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT); + + if (unmap) { + dma_addr_t dma_dest[2]; + struct device *dev = dma->dev; + struct dma_async_tx_descriptor *tx; + enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P; + + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; + unmap->addr[0] = dma_map_page(dev, src, s_off, + len, DMA_TO_DEVICE); + unmap->to_cnt++; + unmap->addr[1] = dma_map_page(dev, dest, d_off, + len, DMA_BIDIRECTIONAL); + dma_dest[1] = unmap->addr[1]; + unmap->bidi_cnt++; + unmap->len = len; + + /* this looks funny, but the engine looks for Q at + * dma_dest[1] and ignores dma_dest[0] as a dest + * due to DMA_PREP_PQ_DISABLE_P + */ + tx = dma->device_prep_dma_pq(chan, dma_dest, unmap->addr, + 1, &coef, len, dma_flags); + + if (tx) { + dma_set_unmap(tx, unmap); + dmaengine_unmap_put(unmap); + async_tx_submit(chan, tx, submit); + return tx; + } + + /* could not get a descriptor, unmap and fall through to + * the synchronous path + */ + dmaengine_unmap_put(unmap); + } + + /* no channel available, or failed to allocate a descriptor, so + * perform the operation synchronously + */ + async_tx_quiesce(&submit->depend_tx); + qmul = raid6_gfmul[coef]; + d = page_address(dest) + d_off; + s = page_address(src) + s_off; + + while (len--) + *d++ = qmul[*s++]; + + return NULL; +} + +static struct dma_async_tx_descriptor * +__2data_recov_4(int disks, size_t bytes, int faila, int failb, + struct page **blocks, unsigned int *offs, + struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct page *p, *q, *a, *b; + unsigned int p_off, q_off, a_off, b_off; + struct page *srcs[2]; + unsigned int src_offs[2]; + unsigned char coef[2]; + enum async_tx_flags flags = submit->flags; + dma_async_tx_callback cb_fn = submit->cb_fn; + void *cb_param = submit->cb_param; + void *scribble = submit->scribble; + + p = blocks[disks-2]; + p_off = offs[disks-2]; + q = blocks[disks-1]; + q_off = offs[disks-1]; + + a = blocks[faila]; + a_off = offs[faila]; + b = blocks[failb]; + b_off = offs[failb]; + + /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */ + /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ + srcs[0] = p; + src_offs[0] = p_off; + srcs[1] = q; + src_offs[1] = q_off; + coef[0] = raid6_gfexi[failb-faila]; + coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_sum_product(b, b_off, srcs, src_offs, coef, bytes, submit); + + /* Dy = P+Pxy+Dx */ + srcs[0] = p; + src_offs[0] = p_off; + srcs[1] = b; + src_offs[1] = b_off; + init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn, + cb_param, scribble); + tx = async_xor_offs(a, a_off, srcs, src_offs, 2, bytes, submit); + + return tx; + +} + +static struct dma_async_tx_descriptor * +__2data_recov_5(int disks, size_t bytes, int faila, int failb, + struct page **blocks, unsigned int *offs, + struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct page *p, *q, *g, *dp, *dq; + unsigned int p_off, q_off, g_off, dp_off, dq_off; + struct page *srcs[2]; + unsigned int src_offs[2]; + unsigned char coef[2]; + enum async_tx_flags flags = submit->flags; + dma_async_tx_callback cb_fn = submit->cb_fn; + void *cb_param = submit->cb_param; + void *scribble = submit->scribble; + int good_srcs, good, i; + + good_srcs = 0; + good = -1; + for (i = 0; i < disks-2; i++) { + if (blocks[i] == NULL) + continue; + if (i == faila || i == failb) + continue; + good = i; + good_srcs++; + } + BUG_ON(good_srcs > 1); + + p = blocks[disks-2]; + p_off = offs[disks-2]; + q = blocks[disks-1]; + q_off = offs[disks-1]; + g = blocks[good]; + g_off = offs[good]; + + /* Compute syndrome with zero for the missing data pages + * Use the dead data pages as temporary storage for delta p and + * delta q + */ + dp = blocks[faila]; + dp_off = offs[faila]; + dq = blocks[failb]; + dq_off = offs[failb]; + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_memcpy(dp, g, dp_off, g_off, bytes, submit); + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_mult(dq, dq_off, g, g_off, + raid6_gfexp[good], bytes, submit); + + /* compute P + Pxy */ + srcs[0] = dp; + src_offs[0] = dp_off; + srcs[1] = p; + src_offs[1] = p_off; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit); + + /* compute Q + Qxy */ + srcs[0] = dq; + src_offs[0] = dq_off; + srcs[1] = q; + src_offs[1] = q_off; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit); + + /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ + srcs[0] = dp; + src_offs[0] = dp_off; + srcs[1] = dq; + src_offs[1] = dq_off; + coef[0] = raid6_gfexi[failb-faila]; + coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_sum_product(dq, dq_off, srcs, src_offs, coef, bytes, submit); + + /* Dy = P+Pxy+Dx */ + srcs[0] = dp; + src_offs[0] = dp_off; + srcs[1] = dq; + src_offs[1] = dq_off; + init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, + cb_param, scribble); + tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit); + + return tx; +} + +static struct dma_async_tx_descriptor * +__2data_recov_n(int disks, size_t bytes, int faila, int failb, + struct page **blocks, unsigned int *offs, + struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct page *p, *q, *dp, *dq; + unsigned int p_off, q_off, dp_off, dq_off; + struct page *srcs[2]; + unsigned int src_offs[2]; + unsigned char coef[2]; + enum async_tx_flags flags = submit->flags; + dma_async_tx_callback cb_fn = submit->cb_fn; + void *cb_param = submit->cb_param; + void *scribble = submit->scribble; + + p = blocks[disks-2]; + p_off = offs[disks-2]; + q = blocks[disks-1]; + q_off = offs[disks-1]; + + /* Compute syndrome with zero for the missing data pages + * Use the dead data pages as temporary storage for + * delta p and delta q + */ + dp = blocks[faila]; + dp_off = offs[faila]; + blocks[faila] = NULL; + blocks[disks-2] = dp; + offs[disks-2] = dp_off; + dq = blocks[failb]; + dq_off = offs[failb]; + blocks[failb] = NULL; + blocks[disks-1] = dq; + offs[disks-1] = dq_off; + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_gen_syndrome(blocks, offs, disks, bytes, submit); + + /* Restore pointer table */ + blocks[faila] = dp; + offs[faila] = dp_off; + blocks[failb] = dq; + offs[failb] = dq_off; + blocks[disks-2] = p; + offs[disks-2] = p_off; + blocks[disks-1] = q; + offs[disks-1] = q_off; + + /* compute P + Pxy */ + srcs[0] = dp; + src_offs[0] = dp_off; + srcs[1] = p; + src_offs[1] = p_off; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit); + + /* compute Q + Qxy */ + srcs[0] = dq; + src_offs[0] = dq_off; + srcs[1] = q; + src_offs[1] = q_off; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit); + + /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ + srcs[0] = dp; + src_offs[0] = dp_off; + srcs[1] = dq; + src_offs[1] = dq_off; + coef[0] = raid6_gfexi[failb-faila]; + coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_sum_product(dq, dq_off, srcs, src_offs, coef, bytes, submit); + + /* Dy = P+Pxy+Dx */ + srcs[0] = dp; + src_offs[0] = dp_off; + srcs[1] = dq; + src_offs[1] = dq_off; + init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, + cb_param, scribble); + tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit); + + return tx; +} + +/** + * async_raid6_2data_recov - asynchronously calculate two missing data blocks + * @disks: number of disks in the RAID-6 array + * @bytes: block size + * @faila: first failed drive index + * @failb: second failed drive index + * @blocks: array of source pointers where the last two entries are p and q + * @offs: array of offset for pages in blocks + * @submit: submission/completion modifiers + */ +struct dma_async_tx_descriptor * +async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb, + struct page **blocks, unsigned int *offs, + struct async_submit_ctl *submit) +{ + void *scribble = submit->scribble; + int non_zero_srcs, i; + + BUG_ON(faila == failb); + if (failb < faila) + swap(faila, failb); + + pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes); + + /* if a dma resource is not available or a scribble buffer is not + * available punt to the synchronous path. In the 'dma not + * available' case be sure to use the scribble buffer to + * preserve the content of 'blocks' as the caller intended. + */ + if (!async_dma_find_channel(DMA_PQ) || !scribble) { + void **ptrs = scribble ? scribble : (void **) blocks; + + async_tx_quiesce(&submit->depend_tx); + for (i = 0; i < disks; i++) + if (blocks[i] == NULL) + ptrs[i] = page_address(ZERO_PAGE(0)); + else + ptrs[i] = page_address(blocks[i]) + offs[i]; + + raid6_recov_2data(disks, bytes, faila, failb, ptrs); + + async_tx_sync_epilog(submit); + + return NULL; + } + + non_zero_srcs = 0; + for (i = 0; i < disks-2 && non_zero_srcs < 4; i++) + if (blocks[i]) + non_zero_srcs++; + switch (non_zero_srcs) { + case 0: + case 1: + /* There must be at least 2 sources - the failed devices. */ + BUG(); + + case 2: + /* dma devices do not uniformly understand a zero source pq + * operation (in contrast to the synchronous case), so + * explicitly handle the special case of a 4 disk array with + * both data disks missing. + */ + return __2data_recov_4(disks, bytes, faila, failb, + blocks, offs, submit); + case 3: + /* dma devices do not uniformly understand a single + * source pq operation (in contrast to the synchronous + * case), so explicitly handle the special case of a 5 disk + * array with 2 of 3 data disks missing. + */ + return __2data_recov_5(disks, bytes, faila, failb, + blocks, offs, submit); + default: + return __2data_recov_n(disks, bytes, faila, failb, + blocks, offs, submit); + } +} +EXPORT_SYMBOL_GPL(async_raid6_2data_recov); + +/** + * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block + * @disks: number of disks in the RAID-6 array + * @bytes: block size + * @faila: failed drive index + * @blocks: array of source pointers where the last two entries are p and q + * @offs: array of offset for pages in blocks + * @submit: submission/completion modifiers + */ +struct dma_async_tx_descriptor * +async_raid6_datap_recov(int disks, size_t bytes, int faila, + struct page **blocks, unsigned int *offs, + struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct page *p, *q, *dq; + unsigned int p_off, q_off, dq_off; + u8 coef; + enum async_tx_flags flags = submit->flags; + dma_async_tx_callback cb_fn = submit->cb_fn; + void *cb_param = submit->cb_param; + void *scribble = submit->scribble; + int good_srcs, good, i; + struct page *srcs[2]; + unsigned int src_offs[2]; + + pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes); + + /* if a dma resource is not available or a scribble buffer is not + * available punt to the synchronous path. In the 'dma not + * available' case be sure to use the scribble buffer to + * preserve the content of 'blocks' as the caller intended. + */ + if (!async_dma_find_channel(DMA_PQ) || !scribble) { + void **ptrs = scribble ? scribble : (void **) blocks; + + async_tx_quiesce(&submit->depend_tx); + for (i = 0; i < disks; i++) + if (blocks[i] == NULL) + ptrs[i] = page_address(ZERO_PAGE(0)); + else + ptrs[i] = page_address(blocks[i]) + offs[i]; + + raid6_recov_datap(disks, bytes, faila, ptrs); + + async_tx_sync_epilog(submit); + + return NULL; + } + + good_srcs = 0; + good = -1; + for (i = 0; i < disks-2; i++) { + if (i == faila) + continue; + if (blocks[i]) { + good = i; + good_srcs++; + if (good_srcs > 1) + break; + } + } + BUG_ON(good_srcs == 0); + + p = blocks[disks-2]; + p_off = offs[disks-2]; + q = blocks[disks-1]; + q_off = offs[disks-1]; + + /* Compute syndrome with zero for the missing data page + * Use the dead data page as temporary storage for delta q + */ + dq = blocks[faila]; + dq_off = offs[faila]; + blocks[faila] = NULL; + blocks[disks-1] = dq; + offs[disks-1] = dq_off; + + /* in the 4-disk case we only need to perform a single source + * multiplication with the one good data block. + */ + if (good_srcs == 1) { + struct page *g = blocks[good]; + unsigned int g_off = offs[good]; + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, + scribble); + tx = async_memcpy(p, g, p_off, g_off, bytes, submit); + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, + scribble); + tx = async_mult(dq, dq_off, g, g_off, + raid6_gfexp[good], bytes, submit); + } else { + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, + scribble); + tx = async_gen_syndrome(blocks, offs, disks, bytes, submit); + } + + /* Restore pointer table */ + blocks[faila] = dq; + offs[faila] = dq_off; + blocks[disks-1] = q; + offs[disks-1] = q_off; + + /* calculate g^{-faila} */ + coef = raid6_gfinv[raid6_gfexp[faila]]; + + srcs[0] = dq; + src_offs[0] = dq_off; + srcs[1] = q; + src_offs[1] = q_off; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit); + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_mult(dq, dq_off, dq, dq_off, coef, bytes, submit); + + srcs[0] = p; + src_offs[0] = p_off; + srcs[1] = dq; + src_offs[1] = dq_off; + init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, + cb_param, scribble); + tx = async_xor_offs(p, p_off, srcs, src_offs, 2, bytes, submit); + + return tx; +} +EXPORT_SYMBOL_GPL(async_raid6_datap_recov); + +MODULE_AUTHOR("Dan Williams "); +MODULE_DESCRIPTION("asynchronous RAID-6 recovery api"); +MODULE_LICENSE("GPL"); diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c new file mode 100644 index 000000000..ad72057a5 --- /dev/null +++ b/crypto/async_tx/async_tx.c @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * core routines for the asynchronous memory transfer/transform api + * + * Copyright © 2006, Intel Corporation. + * + * Dan Williams + * + * with architecture considerations by: + * Neil Brown + * Jeff Garzik + */ +#include +#include +#include +#include + +#ifdef CONFIG_DMA_ENGINE +static int __init async_tx_init(void) +{ + async_dmaengine_get(); + + printk(KERN_INFO "async_tx: api initialized (async)\n"); + + return 0; +} + +static void __exit async_tx_exit(void) +{ + async_dmaengine_put(); +} + +module_init(async_tx_init); +module_exit(async_tx_exit); + +/** + * __async_tx_find_channel - find a channel to carry out the operation or let + * the transaction execute synchronously + * @submit: transaction dependency and submission modifiers + * @tx_type: transaction type + */ +struct dma_chan * +__async_tx_find_channel(struct async_submit_ctl *submit, + enum dma_transaction_type tx_type) +{ + struct dma_async_tx_descriptor *depend_tx = submit->depend_tx; + + /* see if we can keep the chain on one channel */ + if (depend_tx && + dma_has_cap(tx_type, depend_tx->chan->device->cap_mask)) + return depend_tx->chan; + return async_dma_find_channel(tx_type); +} +EXPORT_SYMBOL_GPL(__async_tx_find_channel); +#endif + + +/** + * async_tx_channel_switch - queue an interrupt descriptor with a dependency + * pre-attached. + * @depend_tx: the operation that must finish before the new operation runs + * @tx: the new operation + */ +static void +async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx, + struct dma_async_tx_descriptor *tx) +{ + struct dma_chan *chan = depend_tx->chan; + struct dma_device *device = chan->device; + struct dma_async_tx_descriptor *intr_tx = (void *) ~0; + + /* first check to see if we can still append to depend_tx */ + txd_lock(depend_tx); + if (txd_parent(depend_tx) && depend_tx->chan == tx->chan) { + txd_chain(depend_tx, tx); + intr_tx = NULL; + } + txd_unlock(depend_tx); + + /* attached dependency, flush the parent channel */ + if (!intr_tx) { + device->device_issue_pending(chan); + return; + } + + /* see if we can schedule an interrupt + * otherwise poll for completion + */ + if (dma_has_cap(DMA_INTERRUPT, device->cap_mask)) + intr_tx = device->device_prep_dma_interrupt(chan, 0); + else + intr_tx = NULL; + + if (intr_tx) { + intr_tx->callback = NULL; + intr_tx->callback_param = NULL; + /* safe to chain outside the lock since we know we are + * not submitted yet + */ + txd_chain(intr_tx, tx); + + /* check if we need to append */ + txd_lock(depend_tx); + if (txd_parent(depend_tx)) { + txd_chain(depend_tx, intr_tx); + async_tx_ack(intr_tx); + intr_tx = NULL; + } + txd_unlock(depend_tx); + + if (intr_tx) { + txd_clear_parent(intr_tx); + intr_tx->tx_submit(intr_tx); + async_tx_ack(intr_tx); + } + device->device_issue_pending(chan); + } else { + if (dma_wait_for_async_tx(depend_tx) != DMA_COMPLETE) + panic("%s: DMA error waiting for depend_tx\n", + __func__); + tx->tx_submit(tx); + } +} + + +/** + * enum submit_disposition - flags for routing an incoming operation + * @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock + * @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch + * @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly + * + * while holding depend_tx->lock we must avoid submitting new operations + * to prevent a circular locking dependency with drivers that already + * hold a channel lock when calling async_tx_run_dependencies. + */ +enum submit_disposition { + ASYNC_TX_SUBMITTED, + ASYNC_TX_CHANNEL_SWITCH, + ASYNC_TX_DIRECT_SUBMIT, +}; + +void +async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx, + struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *depend_tx = submit->depend_tx; + + tx->callback = submit->cb_fn; + tx->callback_param = submit->cb_param; + + if (depend_tx) { + enum submit_disposition s; + + /* sanity check the dependency chain: + * 1/ if ack is already set then we cannot be sure + * we are referring to the correct operation + * 2/ dependencies are 1:1 i.e. two transactions can + * not depend on the same parent + */ + BUG_ON(async_tx_test_ack(depend_tx) || txd_next(depend_tx) || + txd_parent(tx)); + + /* the lock prevents async_tx_run_dependencies from missing + * the setting of ->next when ->parent != NULL + */ + txd_lock(depend_tx); + if (txd_parent(depend_tx)) { + /* we have a parent so we can not submit directly + * if we are staying on the same channel: append + * else: channel switch + */ + if (depend_tx->chan == chan) { + txd_chain(depend_tx, tx); + s = ASYNC_TX_SUBMITTED; + } else + s = ASYNC_TX_CHANNEL_SWITCH; + } else { + /* we do not have a parent so we may be able to submit + * directly if we are staying on the same channel + */ + if (depend_tx->chan == chan) + s = ASYNC_TX_DIRECT_SUBMIT; + else + s = ASYNC_TX_CHANNEL_SWITCH; + } + txd_unlock(depend_tx); + + switch (s) { + case ASYNC_TX_SUBMITTED: + break; + case ASYNC_TX_CHANNEL_SWITCH: + async_tx_channel_switch(depend_tx, tx); + break; + case ASYNC_TX_DIRECT_SUBMIT: + txd_clear_parent(tx); + tx->tx_submit(tx); + break; + } + } else { + txd_clear_parent(tx); + tx->tx_submit(tx); + } + + if (submit->flags & ASYNC_TX_ACK) + async_tx_ack(tx); + + if (depend_tx) + async_tx_ack(depend_tx); +} +EXPORT_SYMBOL_GPL(async_tx_submit); + +/** + * async_trigger_callback - schedules the callback function to be run + * @submit: submission and completion parameters + * + * honored flags: ASYNC_TX_ACK + * + * The callback is run after any dependent operations have completed. + */ +struct dma_async_tx_descriptor * +async_trigger_callback(struct async_submit_ctl *submit) +{ + struct dma_chan *chan; + struct dma_device *device; + struct dma_async_tx_descriptor *tx; + struct dma_async_tx_descriptor *depend_tx = submit->depend_tx; + + if (depend_tx) { + chan = depend_tx->chan; + device = chan->device; + + /* see if we can schedule an interrupt + * otherwise poll for completion + */ + if (device && !dma_has_cap(DMA_INTERRUPT, device->cap_mask)) + device = NULL; + + tx = device ? device->device_prep_dma_interrupt(chan, 0) : NULL; + } else + tx = NULL; + + if (tx) { + pr_debug("%s: (async)\n", __func__); + + async_tx_submit(chan, tx, submit); + } else { + pr_debug("%s: (sync)\n", __func__); + + /* wait for any prerequisite operations */ + async_tx_quiesce(&submit->depend_tx); + + async_tx_sync_epilog(submit); + } + + return tx; +} +EXPORT_SYMBOL_GPL(async_trigger_callback); + +/** + * async_tx_quiesce - ensure tx is complete and freeable upon return + * @tx: transaction to quiesce + */ +void async_tx_quiesce(struct dma_async_tx_descriptor **tx) +{ + if (*tx) { + /* if ack is already set then we cannot be sure + * we are referring to the correct operation + */ + BUG_ON(async_tx_test_ack(*tx)); + if (dma_wait_for_async_tx(*tx) != DMA_COMPLETE) + panic("%s: DMA error waiting for transaction\n", + __func__); + async_tx_ack(*tx); + *tx = NULL; + } +} +EXPORT_SYMBOL_GPL(async_tx_quiesce); + +MODULE_AUTHOR("Intel Corporation"); +MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API"); +MODULE_LICENSE("GPL"); diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c new file mode 100644 index 000000000..84458375b --- /dev/null +++ b/crypto/async_tx/async_xor.c @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * xor offload engine api + * + * Copyright © 2006, Intel Corporation. + * + * Dan Williams + * + * with architecture considerations by: + * Neil Brown + * Jeff Garzik + */ +#include +#include +#include +#include +#include +#include +#include + +/* do_async_xor - dma map the pages and perform the xor with an engine */ +static __async_inline struct dma_async_tx_descriptor * +do_async_xor(struct dma_chan *chan, struct dmaengine_unmap_data *unmap, + struct async_submit_ctl *submit) +{ + struct dma_device *dma = chan->device; + struct dma_async_tx_descriptor *tx = NULL; + dma_async_tx_callback cb_fn_orig = submit->cb_fn; + void *cb_param_orig = submit->cb_param; + enum async_tx_flags flags_orig = submit->flags; + enum dma_ctrl_flags dma_flags = 0; + int src_cnt = unmap->to_cnt; + int xor_src_cnt; + dma_addr_t dma_dest = unmap->addr[unmap->to_cnt]; + dma_addr_t *src_list = unmap->addr; + + while (src_cnt) { + dma_addr_t tmp; + + submit->flags = flags_orig; + xor_src_cnt = min(src_cnt, (int)dma->max_xor); + /* if we are submitting additional xors, leave the chain open + * and clear the callback parameters + */ + if (src_cnt > xor_src_cnt) { + submit->flags &= ~ASYNC_TX_ACK; + submit->flags |= ASYNC_TX_FENCE; + submit->cb_fn = NULL; + submit->cb_param = NULL; + } else { + submit->cb_fn = cb_fn_orig; + submit->cb_param = cb_param_orig; + } + if (submit->cb_fn) + dma_flags |= DMA_PREP_INTERRUPT; + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; + + /* Drivers force forward progress in case they can not provide a + * descriptor + */ + tmp = src_list[0]; + if (src_list > unmap->addr) + src_list[0] = dma_dest; + tx = dma->device_prep_dma_xor(chan, dma_dest, src_list, + xor_src_cnt, unmap->len, + dma_flags); + + if (unlikely(!tx)) + async_tx_quiesce(&submit->depend_tx); + + /* spin wait for the preceding transactions to complete */ + while (unlikely(!tx)) { + dma_async_issue_pending(chan); + tx = dma->device_prep_dma_xor(chan, dma_dest, + src_list, + xor_src_cnt, unmap->len, + dma_flags); + } + src_list[0] = tmp; + + dma_set_unmap(tx, unmap); + async_tx_submit(chan, tx, submit); + submit->depend_tx = tx; + + if (src_cnt > xor_src_cnt) { + /* drop completed sources */ + src_cnt -= xor_src_cnt; + /* use the intermediate result a source */ + src_cnt++; + src_list += xor_src_cnt - 1; + } else + break; + } + + return tx; +} + +static void +do_sync_xor_offs(struct page *dest, unsigned int offset, + struct page **src_list, unsigned int *src_offs, + int src_cnt, size_t len, struct async_submit_ctl *submit) +{ + int i; + int xor_src_cnt = 0; + void *dest_buf; + void **srcs; + + if (submit->scribble) + srcs = submit->scribble; + else + srcs = (void **) src_list; + + /* convert to buffer pointers */ + for (i = 0; i < src_cnt; i++) + if (src_list[i]) + srcs[xor_src_cnt++] = page_address(src_list[i]) + + (src_offs ? src_offs[i] : offset); + + /* set destination address */ + dest_buf = page_address(dest) + offset; + if (submit->flags & ASYNC_TX_XOR_ZERO_DST) + memset(dest_buf, 0, len); + xor_gen(dest_buf, srcs, xor_src_cnt, len); + async_tx_sync_epilog(submit); +} + +static inline bool +dma_xor_aligned_offsets(struct dma_device *device, unsigned int offset, + unsigned int *src_offs, int src_cnt, int len) +{ + int i; + + if (!is_dma_xor_aligned(device, offset, 0, len)) + return false; + + if (!src_offs) + return true; + + for (i = 0; i < src_cnt; i++) { + if (!is_dma_xor_aligned(device, src_offs[i], 0, len)) + return false; + } + return true; +} + +/** + * async_xor_offs - attempt to xor a set of blocks with a dma engine. + * @dest: destination page + * @offset: dst offset to start transaction + * @src_list: array of source pages + * @src_offs: array of source pages offset, NULL means common src/dst offset + * @src_cnt: number of source pages + * @len: length in bytes + * @submit: submission / completion modifiers + * + * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST + * + * xor_gen always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST flag + * must be set to not include dest data in the calculation. The assumption with + * dma engines is that they only use the destination buffer as a source when it + * is explicitly specified in the source list. + * + * src_list note: if the dest is also a source it must be at index zero. + * The contents of this array will be overwritten if a scribble region + * is not specified. + */ +struct dma_async_tx_descriptor * +async_xor_offs(struct page *dest, unsigned int offset, + struct page **src_list, unsigned int *src_offs, + int src_cnt, size_t len, struct async_submit_ctl *submit) +{ + struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR, + &dest, 1, src_list, + src_cnt, len); + struct dma_device *device = chan ? chan->device : NULL; + struct dmaengine_unmap_data *unmap = NULL; + + BUG_ON(src_cnt <= 1); + + if (device) + unmap = dmaengine_get_unmap_data(device->dev, src_cnt+1, GFP_NOWAIT); + + if (unmap && dma_xor_aligned_offsets(device, offset, + src_offs, src_cnt, len)) { + struct dma_async_tx_descriptor *tx; + int i, j; + + /* run the xor asynchronously */ + pr_debug("%s (async): len: %zu\n", __func__, len); + + unmap->len = len; + for (i = 0, j = 0; i < src_cnt; i++) { + if (!src_list[i]) + continue; + unmap->to_cnt++; + unmap->addr[j++] = dma_map_page(device->dev, src_list[i], + src_offs ? src_offs[i] : offset, + len, DMA_TO_DEVICE); + } + + /* map it bidirectional as it may be re-used as a source */ + unmap->addr[j] = dma_map_page(device->dev, dest, offset, len, + DMA_BIDIRECTIONAL); + unmap->bidi_cnt = 1; + + tx = do_async_xor(chan, unmap, submit); + dmaengine_unmap_put(unmap); + return tx; + } else { + dmaengine_unmap_put(unmap); + /* run the xor synchronously */ + pr_debug("%s (sync): len: %zu\n", __func__, len); + WARN_ONCE(chan, "%s: no space for dma address conversion\n", + __func__); + + /* in the sync case the dest is an implied source + * (assumes the dest is the first source) + */ + if (submit->flags & ASYNC_TX_XOR_DROP_DST) { + src_cnt--; + src_list++; + if (src_offs) + src_offs++; + } + + /* wait for any prerequisite operations */ + async_tx_quiesce(&submit->depend_tx); + + do_sync_xor_offs(dest, offset, src_list, src_offs, + src_cnt, len, submit); + + return NULL; + } +} +EXPORT_SYMBOL_GPL(async_xor_offs); + +/** + * async_xor - attempt to xor a set of blocks with a dma engine. + * @dest: destination page + * @src_list: array of source pages + * @offset: common src/dst offset to start transaction + * @src_cnt: number of source pages + * @len: length in bytes + * @submit: submission / completion modifiers + * + * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST + * + * xor_gen always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST flag + * must be set to not include dest data in the calculation. The assumption with + * dma engines is that they only use the destination buffer as a source when it + * is explicitly specified in the source list. + * + * src_list note: if the dest is also a source it must be at index zero. + * The contents of this array will be overwritten if a scribble region + * is not specified. + */ +struct dma_async_tx_descriptor * +async_xor(struct page *dest, struct page **src_list, unsigned int offset, + int src_cnt, size_t len, struct async_submit_ctl *submit) +{ + return async_xor_offs(dest, offset, src_list, NULL, + src_cnt, len, submit); +} +EXPORT_SYMBOL_GPL(async_xor); + +static int page_is_zero(struct page *p, unsigned int offset, size_t len) +{ + return !memchr_inv(page_address(p) + offset, 0, len); +} + +static inline struct dma_chan * +xor_val_chan(struct async_submit_ctl *submit, struct page *dest, + struct page **src_list, int src_cnt, size_t len) +{ + #ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA + return NULL; + #endif + return async_tx_find_channel(submit, DMA_XOR_VAL, &dest, 1, src_list, + src_cnt, len); +} + +/** + * async_xor_val_offs - attempt a xor parity check with a dma engine. + * @dest: destination page used if the xor is performed synchronously + * @offset: des offset in pages to start transaction + * @src_list: array of source pages + * @src_offs: array of source pages offset, NULL means common src/det offset + * @src_cnt: number of source pages + * @len: length in bytes + * @result: 0 if sum == 0 else non-zero + * @submit: submission / completion modifiers + * + * honored flags: ASYNC_TX_ACK + * + * src_list note: if the dest is also a source it must be at index zero. + * The contents of this array will be overwritten if a scribble region + * is not specified. + */ +struct dma_async_tx_descriptor * +async_xor_val_offs(struct page *dest, unsigned int offset, + struct page **src_list, unsigned int *src_offs, + int src_cnt, size_t len, enum sum_check_flags *result, + struct async_submit_ctl *submit) +{ + struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len); + struct dma_device *device = chan ? chan->device : NULL; + struct dma_async_tx_descriptor *tx = NULL; + struct dmaengine_unmap_data *unmap = NULL; + + BUG_ON(src_cnt <= 1); + + if (device) + unmap = dmaengine_get_unmap_data(device->dev, src_cnt, GFP_NOWAIT); + + if (unmap && src_cnt <= device->max_xor && + dma_xor_aligned_offsets(device, offset, src_offs, src_cnt, len)) { + unsigned long dma_prep_flags = 0; + int i; + + pr_debug("%s: (async) len: %zu\n", __func__, len); + + if (submit->cb_fn) + dma_prep_flags |= DMA_PREP_INTERRUPT; + if (submit->flags & ASYNC_TX_FENCE) + dma_prep_flags |= DMA_PREP_FENCE; + + for (i = 0; i < src_cnt; i++) { + unmap->addr[i] = dma_map_page(device->dev, src_list[i], + src_offs ? src_offs[i] : offset, + len, DMA_TO_DEVICE); + unmap->to_cnt++; + } + unmap->len = len; + + tx = device->device_prep_dma_xor_val(chan, unmap->addr, src_cnt, + len, result, + dma_prep_flags); + if (unlikely(!tx)) { + async_tx_quiesce(&submit->depend_tx); + + while (!tx) { + dma_async_issue_pending(chan); + tx = device->device_prep_dma_xor_val(chan, + unmap->addr, src_cnt, len, result, + dma_prep_flags); + } + } + dma_set_unmap(tx, unmap); + async_tx_submit(chan, tx, submit); + } else { + enum async_tx_flags flags_orig = submit->flags; + + pr_debug("%s: (sync) len: %zu\n", __func__, len); + WARN_ONCE(device && src_cnt <= device->max_xor, + "%s: no space for dma address conversion\n", + __func__); + + submit->flags |= ASYNC_TX_XOR_DROP_DST; + submit->flags &= ~ASYNC_TX_ACK; + + tx = async_xor_offs(dest, offset, src_list, src_offs, + src_cnt, len, submit); + + async_tx_quiesce(&tx); + + *result = !page_is_zero(dest, offset, len) << SUM_CHECK_P; + + async_tx_sync_epilog(submit); + submit->flags = flags_orig; + } + dmaengine_unmap_put(unmap); + + return tx; +} +EXPORT_SYMBOL_GPL(async_xor_val_offs); + +MODULE_AUTHOR("Intel Corporation"); +MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api"); +MODULE_LICENSE("GPL"); diff --git a/crypto/async_tx/raid6test.c b/crypto/async_tx/raid6test.c new file mode 100644 index 000000000..d3fbee1e0 --- /dev/null +++ b/crypto/async_tx/raid6test.c @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * asynchronous raid6 recovery self test + * Copyright (c) 2009, Intel Corporation. + * + * based on drivers/md/raid6test/test.c: + * Copyright 2002-2007 H. Peter Anvin + */ +#include +#include +#include +#include +#include + +#undef pr +#define pr(fmt, args...) pr_info("raid6test: " fmt, ##args) + +#define NDISKS 64 /* Including P and Q */ + +static struct page *dataptrs[NDISKS]; +unsigned int dataoffs[NDISKS]; +static addr_conv_t addr_conv[NDISKS]; +static struct page *data[NDISKS+3]; +static struct page *spare; +static struct page *recovi; +static struct page *recovj; + +static void callback(void *param) +{ + struct completion *cmp = param; + + complete(cmp); +} + +static void makedata(int disks) +{ + int i; + + for (i = 0; i < disks; i++) { + get_random_bytes(page_address(data[i]), PAGE_SIZE); + dataptrs[i] = data[i]; + dataoffs[i] = 0; + } +} + +static char disk_type(int d, int disks) +{ + if (d == disks - 2) + return 'P'; + else if (d == disks - 1) + return 'Q'; + else + return 'D'; +} + +/* Recover two failed blocks. */ +static void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, + struct page **ptrs, unsigned int *offs) +{ + struct async_submit_ctl submit; + struct completion cmp; + struct dma_async_tx_descriptor *tx = NULL; + enum sum_check_flags result = ~0; + + if (faila > failb) + swap(faila, failb); + + if (failb == disks-1) { + if (faila == disks-2) { + /* P+Q failure. Just rebuild the syndrome. */ + init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv); + tx = async_gen_syndrome(ptrs, offs, + disks, bytes, &submit); + } else { + struct page *blocks[NDISKS]; + struct page *dest; + int count = 0; + int i; + + BUG_ON(disks > NDISKS); + + /* data+Q failure. Reconstruct data from P, + * then rebuild syndrome + */ + for (i = disks; i-- ; ) { + if (i == faila || i == failb) + continue; + blocks[count++] = ptrs[i]; + } + dest = ptrs[faila]; + init_async_submit(&submit, ASYNC_TX_XOR_ZERO_DST, NULL, + NULL, NULL, addr_conv); + tx = async_xor(dest, blocks, 0, count, bytes, &submit); + + init_async_submit(&submit, 0, tx, NULL, NULL, addr_conv); + tx = async_gen_syndrome(ptrs, offs, + disks, bytes, &submit); + } + } else { + if (failb == disks-2) { + /* data+P failure. */ + init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv); + tx = async_raid6_datap_recov(disks, bytes, + faila, ptrs, offs, &submit); + } else { + /* data+data failure. */ + init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv); + tx = async_raid6_2data_recov(disks, bytes, + faila, failb, ptrs, offs, &submit); + } + } + init_completion(&cmp); + init_async_submit(&submit, ASYNC_TX_ACK, tx, callback, &cmp, addr_conv); + tx = async_syndrome_val(ptrs, offs, + disks, bytes, &result, spare, 0, &submit); + async_tx_issue_pending(tx); + + if (wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)) == 0) + pr("%s: timeout! (faila: %d failb: %d disks: %d)\n", + __func__, faila, failb, disks); + + if (result != 0) + pr("%s: validation failure! faila: %d failb: %d sum_check_flags: %x\n", + __func__, faila, failb, result); +} + +static int test_disks(int i, int j, int disks) +{ + int erra, errb; + + memset(page_address(recovi), 0xf0, PAGE_SIZE); + memset(page_address(recovj), 0xba, PAGE_SIZE); + + dataptrs[i] = recovi; + dataptrs[j] = recovj; + + raid6_dual_recov(disks, PAGE_SIZE, i, j, dataptrs, dataoffs); + + erra = memcmp(page_address(data[i]), page_address(recovi), PAGE_SIZE); + errb = memcmp(page_address(data[j]), page_address(recovj), PAGE_SIZE); + + pr("%s(%d, %d): faila=%3d(%c) failb=%3d(%c) %s\n", + __func__, i, j, i, disk_type(i, disks), j, disk_type(j, disks), + (!erra && !errb) ? "OK" : !erra ? "ERRB" : !errb ? "ERRA" : "ERRAB"); + + dataptrs[i] = data[i]; + dataptrs[j] = data[j]; + + return erra || errb; +} + +static int test(int disks, int *tests) +{ + struct dma_async_tx_descriptor *tx; + struct async_submit_ctl submit; + struct completion cmp; + int err = 0; + int i, j; + + recovi = data[disks]; + recovj = data[disks+1]; + spare = data[disks+2]; + + makedata(disks); + + /* Nuke syndromes */ + memset(page_address(data[disks-2]), 0xee, PAGE_SIZE); + memset(page_address(data[disks-1]), 0xee, PAGE_SIZE); + + /* Generate assumed good syndrome */ + init_completion(&cmp); + init_async_submit(&submit, ASYNC_TX_ACK, NULL, callback, &cmp, addr_conv); + tx = async_gen_syndrome(dataptrs, dataoffs, disks, PAGE_SIZE, &submit); + async_tx_issue_pending(tx); + + if (wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)) == 0) { + pr("error: initial gen_syndrome(%d) timed out\n", disks); + return 1; + } + + pr("testing the %d-disk case...\n", disks); + for (i = 0; i < disks-1; i++) + for (j = i+1; j < disks; j++) { + (*tests)++; + err += test_disks(i, j, disks); + } + + return err; +} + + +static int __init raid6_test(void) +{ + int err = 0; + int tests = 0; + int i; + + for (i = 0; i < NDISKS+3; i++) { + data[i] = alloc_page(GFP_KERNEL); + if (!data[i]) { + while (i--) + put_page(data[i]); + return -ENOMEM; + } + } + + /* the 4-disk and 5-disk cases are special for the recovery code */ + if (NDISKS > 4) + err += test(4, &tests); + if (NDISKS > 5) + err += test(5, &tests); + /* the 11 and 12 disk cases are special for ioatdma (p-disabled + * q-continuation without extended descriptor) + */ + if (NDISKS > 12) { + err += test(11, &tests); + err += test(12, &tests); + } + + /* the 24 disk case is special for ioatdma as it is the boundary point + * at which it needs to switch from 8-source ops to 16-source + * ops for continuation (assumes DMA_HAS_PQ_CONTINUE is not set) + */ + if (NDISKS > 24) + err += test(24, &tests); + + err += test(NDISKS, &tests); + + pr("\n"); + pr("complete (%d tests, %d failure%s)\n", + tests, err, err == 1 ? "" : "s"); + + for (i = 0; i < NDISKS+3; i++) + put_page(data[i]); + + return 0; +} + +static void __exit raid6_test_exit(void) +{ +} + +/* when compiled-in wait for drivers to load first (assumes dma drivers + * are also compiled-in) + */ +late_initcall(raid6_test); +module_exit(raid6_test_exit); +MODULE_AUTHOR("Dan Williams "); +MODULE_DESCRIPTION("asynchronous RAID-6 recovery self tests"); +MODULE_LICENSE("GPL"); -- cgit v1.3.1