lkml.org 
[lkml]   [2009]   [Sep]   [3]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Subject[PATCH 09/29] ioat1: move descriptor allocation from submit to prep
    From
    Date
    The async_tx api assumes that after a successful ->prep a subsequent
    ->submit will not fail due to a lack of resources.

    This also fixes a bug in the allocation failure case. Previously the
    descriptors allocated prior to the allocation failure would not be
    returned to the free list.

    Signed-off-by: Maciej Sosnowski <maciej.sosnowski@intel.com>
    Signed-off-by: Dan Williams <dan.j.williams@intel.com>
    ---
    drivers/dma/ioat/dma.c | 154 ++++++++++++++++++++----------------------------
    1 files changed, 65 insertions(+), 89 deletions(-)

    diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
    index 4840d48..c4333be 100644
    --- a/drivers/dma/ioat/dma.c
    +++ b/drivers/dma/ioat/dma.c
    @@ -420,95 +420,29 @@ static void ioat_dma_chan_watchdog(struct work_struct *work)
    static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
    {
    struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
    - struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
    - struct ioat_desc_sw *prev, *new;
    - struct ioat_dma_descriptor *hw;
    + struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
    + struct ioat_desc_sw *first;
    + struct ioat_desc_sw *chain_tail;
    dma_cookie_t cookie;
    - LIST_HEAD(new_chain);
    - u32 copy;
    - size_t len;
    - dma_addr_t src, dst;
    - unsigned long orig_flags;
    - unsigned int desc_count = 0;
    -
    - /* src and dest and len are stored in the initial descriptor */
    - len = first->len;
    - src = first->src;
    - dst = first->dst;
    - orig_flags = first->txd.flags;
    - new = first;

    spin_lock_bh(&ioat_chan->desc_lock);
    - prev = to_ioat_desc(ioat_chan->used_desc.prev);
    - prefetch(prev->hw);
    - do {
    - copy = min_t(size_t, len, ioat_chan->xfercap);
    -
    - async_tx_ack(&new->txd);
    -
    - hw = new->hw;
    - hw->size = copy;
    - hw->ctl = 0;
    - hw->src_addr = src;
    - hw->dst_addr = dst;
    - hw->next = 0;
    -
    - /* chain together the physical address list for the HW */
    - wmb();
    - prev->hw->next = (u64) new->txd.phys;
    -
    - len -= copy;
    - dst += copy;
    - src += copy;
    -
    - list_add_tail(&new->node, &new_chain);
    - desc_count++;
    - prev = new;
    - } while (len && (new = ioat1_dma_get_next_descriptor(ioat_chan)));
    -
    - if (!new) {
    - dev_err(to_dev(ioat_chan), "tx submit failed\n");
    - spin_unlock_bh(&ioat_chan->desc_lock);
    - return -ENOMEM;
    - }
    -
    - hw->ctl_f.compl_write = 1;
    - if (first->txd.callback) {
    - hw->ctl_f.int_en = 1;
    - if (first != new) {
    - /* move callback into to last desc */
    - new->txd.callback = first->txd.callback;
    - new->txd.callback_param
    - = first->txd.callback_param;
    - first->txd.callback = NULL;
    - first->txd.callback_param = NULL;
    - }
    - }
    -
    - new->tx_cnt = desc_count;
    - new->txd.flags = orig_flags; /* client is in control of this ack */
    -
    - /* store the original values for use in later cleanup */
    - if (new != first) {
    - new->src = first->src;
    - new->dst = first->dst;
    - new->len = first->len;
    - }
    -
    /* cookie incr and addition to used_list must be atomic */
    cookie = ioat_chan->common.cookie;
    cookie++;
    if (cookie < 0)
    cookie = 1;
    - ioat_chan->common.cookie = new->txd.cookie = cookie;
    + ioat_chan->common.cookie = tx->cookie = cookie;

    /* write address into NextDescriptor field of last desc in chain */
    - to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
    - first->txd.phys;
    - list_splice_tail(&new_chain, &ioat_chan->used_desc);
    -
    - ioat_chan->dmacount += desc_count;
    - ioat_chan->pending += desc_count;
    + first = to_ioat_desc(tx->tx_list.next);
    + chain_tail = to_ioat_desc(ioat_chan->used_desc.prev);
    + /* make descriptor updates globally visible before chaining */
    + wmb();
    + chain_tail->hw->next = first->txd.phys;
    + list_splice_tail_init(&tx->tx_list, &ioat_chan->used_desc);
    +
    + ioat_chan->dmacount += desc->tx_cnt;
    + ioat_chan->pending += desc->tx_cnt;
    if (ioat_chan->pending >= ioat_pending_level)
    __ioat1_dma_memcpy_issue_pending(ioat_chan);
    spin_unlock_bh(&ioat_chan->desc_lock);
    @@ -937,24 +871,66 @@ ioat1_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
    dma_addr_t dma_src, size_t len, unsigned long flags)
    {
    struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
    - struct ioat_desc_sw *new;
    + struct ioat_desc_sw *desc;
    + size_t copy;
    + LIST_HEAD(chain);
    + dma_addr_t src = dma_src;
    + dma_addr_t dest = dma_dest;
    + size_t total_len = len;
    + struct ioat_dma_descriptor *hw = NULL;
    + int tx_cnt = 0;

    spin_lock_bh(&ioat_chan->desc_lock);
    - new = ioat_dma_get_next_descriptor(ioat_chan);
    - spin_unlock_bh(&ioat_chan->desc_lock);
    + desc = ioat_dma_get_next_descriptor(ioat_chan);
    + do {
    + if (!desc)
    + break;

    - if (new) {
    - new->len = len;
    - new->dst = dma_dest;
    - new->src = dma_src;
    - new->txd.flags = flags;
    - return &new->txd;
    - } else {
    + tx_cnt++;
    + copy = min_t(size_t, len, ioat_chan->xfercap);
    +
    + hw = desc->hw;
    + hw->size = copy;
    + hw->ctl = 0;
    + hw->src_addr = src;
    + hw->dst_addr = dest;
    +
    + list_add_tail(&desc->node, &chain);
    +
    + len -= copy;
    + dest += copy;
    + src += copy;
    + if (len) {
    + struct ioat_desc_sw *next;
    +
    + async_tx_ack(&desc->txd);
    + next = ioat_dma_get_next_descriptor(ioat_chan);
    + hw->next = next ? next->txd.phys : 0;
    + desc = next;
    + } else
    + hw->next = 0;
    + } while (len);
    +
    + if (!desc) {
    dev_err(to_dev(ioat_chan),
    "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
    chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
    + list_splice(&chain, &ioat_chan->free_desc);
    + spin_unlock_bh(&ioat_chan->desc_lock);
    return NULL;
    }
    + spin_unlock_bh(&ioat_chan->desc_lock);
    +
    + desc->txd.flags = flags;
    + desc->tx_cnt = tx_cnt;
    + desc->src = dma_src;
    + desc->dst = dma_dest;
    + desc->len = total_len;
    + list_splice(&chain, &desc->txd.tx_list);
    + hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
    + hw->ctl_f.compl_write = 1;
    +
    + return &desc->txd;
    }

    static struct dma_async_tx_descriptor *


    \
     
     \ /
      Last update: 2009-09-04 04:47    [W:4.253 / U:0.104 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site