lkml.org 
[lkml]   [2021]   [Feb]   [11]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH 5.10 05/54] io_uring: always batch cancel in *cancel_files()
    Date
    From: Pavel Begunkov <asml.silence@gmail.com>

    [ Upstream commit f6edbabb8359798c541b0776616c5eab3a840d3d ]

    Instead of iterating over each request and cancelling it individually in
    io_uring_cancel_files(), try to cancel all matching requests and use
    ->inflight_list only to check if there anything left.

    In many cases it should be faster, and we can reuse a lot of code from
    task cancellation.

    Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
    Signed-off-by: Jens Axboe <axboe@kernel.dk>
    Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
    ---
    fs/io-wq.c | 10 ----
    fs/io-wq.h | 1
    fs/io_uring.c | 139 ++++++++--------------------------------------------------
    3 files changed, 20 insertions(+), 130 deletions(-)

    --- a/fs/io-wq.c
    +++ b/fs/io-wq.c
    @@ -1078,16 +1078,6 @@ enum io_wq_cancel io_wq_cancel_cb(struct
    return IO_WQ_CANCEL_NOTFOUND;
    }

    -static bool io_wq_io_cb_cancel_data(struct io_wq_work *work, void *data)
    -{
    - return work == data;
    -}
    -
    -enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork)
    -{
    - return io_wq_cancel_cb(wq, io_wq_io_cb_cancel_data, (void *)cwork, false);
    -}
    -
    struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
    {
    int ret = -ENOMEM, node;
    --- a/fs/io-wq.h
    +++ b/fs/io-wq.h
    @@ -130,7 +130,6 @@ static inline bool io_wq_is_hashed(struc
    }

    void io_wq_cancel_all(struct io_wq *wq);
    -enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork);

    typedef bool (work_cancel_fn)(struct io_wq_work *, void *);

    --- a/fs/io_uring.c
    +++ b/fs/io_uring.c
    @@ -1496,15 +1496,6 @@ static void io_kill_timeout(struct io_ki
    }
    }

    -static bool io_task_match(struct io_kiocb *req, struct task_struct *tsk)
    -{
    - struct io_ring_ctx *ctx = req->ctx;
    -
    - if (!tsk || req->task == tsk)
    - return true;
    - return (ctx->flags & IORING_SETUP_SQPOLL);
    -}
    -
    /*
    * Returns true if we found and killed one or more timeouts
    */
    @@ -8524,112 +8515,31 @@ static int io_uring_release(struct inode
    return 0;
    }

    -/*
    - * Returns true if 'preq' is the link parent of 'req'
    - */
    -static bool io_match_link(struct io_kiocb *preq, struct io_kiocb *req)
    -{
    - struct io_kiocb *link;
    -
    - if (!(preq->flags & REQ_F_LINK_HEAD))
    - return false;
    -
    - list_for_each_entry(link, &preq->link_list, link_list) {
    - if (link == req)
    - return true;
    - }
    -
    - return false;
    -}
    -
    -/*
    - * We're looking to cancel 'req' because it's holding on to our files, but
    - * 'req' could be a link to another request. See if it is, and cancel that
    - * parent request if so.
    - */
    -static bool io_poll_remove_link(struct io_ring_ctx *ctx, struct io_kiocb *req)
    -{
    - struct hlist_node *tmp;
    - struct io_kiocb *preq;
    - bool found = false;
    - int i;
    -
    - spin_lock_irq(&ctx->completion_lock);
    - for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
    - struct hlist_head *list;
    -
    - list = &ctx->cancel_hash[i];
    - hlist_for_each_entry_safe(preq, tmp, list, hash_node) {
    - found = io_match_link(preq, req);
    - if (found) {
    - io_poll_remove_one(preq);
    - break;
    - }
    - }
    - }
    - spin_unlock_irq(&ctx->completion_lock);
    - return found;
    -}
    -
    -static bool io_timeout_remove_link(struct io_ring_ctx *ctx,
    - struct io_kiocb *req)
    -{
    - struct io_kiocb *preq;
    - bool found = false;
    -
    - spin_lock_irq(&ctx->completion_lock);
    - list_for_each_entry(preq, &ctx->timeout_list, timeout.list) {
    - found = io_match_link(preq, req);
    - if (found) {
    - __io_timeout_cancel(preq);
    - break;
    - }
    - }
    - spin_unlock_irq(&ctx->completion_lock);
    - return found;
    -}
    +struct io_task_cancel {
    + struct task_struct *task;
    + struct files_struct *files;
    +};

    -static bool io_cancel_link_cb(struct io_wq_work *work, void *data)
    +static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
    {
    struct io_kiocb *req = container_of(work, struct io_kiocb, work);
    + struct io_task_cancel *cancel = data;
    bool ret;

    - if (req->flags & REQ_F_LINK_TIMEOUT) {
    + if (cancel->files && (req->flags & REQ_F_LINK_TIMEOUT)) {
    unsigned long flags;
    struct io_ring_ctx *ctx = req->ctx;

    /* protect against races with linked timeouts */
    spin_lock_irqsave(&ctx->completion_lock, flags);
    - ret = io_match_link(req, data);
    + ret = io_match_task(req, cancel->task, cancel->files);
    spin_unlock_irqrestore(&ctx->completion_lock, flags);
    } else {
    - ret = io_match_link(req, data);
    + ret = io_match_task(req, cancel->task, cancel->files);
    }
    return ret;
    }

    -static void io_attempt_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req)
    -{
    - enum io_wq_cancel cret;
    -
    - /* cancel this particular work, if it's running */
    - cret = io_wq_cancel_work(ctx->io_wq, &req->work);
    - if (cret != IO_WQ_CANCEL_NOTFOUND)
    - return;
    -
    - /* find links that hold this pending, cancel those */
    - cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_link_cb, req, true);
    - if (cret != IO_WQ_CANCEL_NOTFOUND)
    - return;
    -
    - /* if we have a poll link holding this pending, cancel that */
    - if (io_poll_remove_link(ctx, req))
    - return;
    -
    - /* final option, timeout link is holding this req pending */
    - io_timeout_remove_link(ctx, req);
    -}
    -
    static void io_cancel_defer_files(struct io_ring_ctx *ctx,
    struct task_struct *task,
    struct files_struct *files)
    @@ -8661,8 +8571,10 @@ static void io_uring_cancel_files(struct
    struct files_struct *files)
    {
    while (!list_empty_careful(&ctx->inflight_list)) {
    - struct io_kiocb *cancel_req = NULL, *req;
    + struct io_task_cancel cancel = { .task = task, .files = NULL, };
    + struct io_kiocb *req;
    DEFINE_WAIT(wait);
    + bool found = false;

    spin_lock_irq(&ctx->inflight_lock);
    list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
    @@ -8670,25 +8582,21 @@ static void io_uring_cancel_files(struct
    (req->work.flags & IO_WQ_WORK_FILES) &&
    req->work.identity->files != files)
    continue;
    - /* req is being completed, ignore */
    - if (!refcount_inc_not_zero(&req->refs))
    - continue;
    - cancel_req = req;
    + found = true;
    break;
    }
    - if (cancel_req)
    + if (found)
    prepare_to_wait(&ctx->inflight_wait, &wait,
    TASK_UNINTERRUPTIBLE);
    spin_unlock_irq(&ctx->inflight_lock);

    /* We need to keep going until we don't find a matching req */
    - if (!cancel_req)
    + if (!found)
    break;
    - /* cancel this request, or head link requests */
    - io_attempt_cancel(ctx, cancel_req);
    - io_cqring_overflow_flush(ctx, true, task, files);

    - io_put_req(cancel_req);
    + io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, &cancel, true);
    + io_poll_remove_all(ctx, task, files);
    + io_kill_timeouts(ctx, task, files);
    /* cancellations _may_ trigger task work */
    io_run_task_work();
    schedule();
    @@ -8696,22 +8604,15 @@ static void io_uring_cancel_files(struct
    }
    }

    -static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
    -{
    - struct io_kiocb *req = container_of(work, struct io_kiocb, work);
    - struct task_struct *task = data;
    -
    - return io_task_match(req, task);
    -}
    -
    static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx,
    struct task_struct *task)
    {
    while (1) {
    + struct io_task_cancel cancel = { .task = task, .files = NULL, };
    enum io_wq_cancel cret;
    bool ret = false;

    - cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, task, true);
    + cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, &cancel, true);
    if (cret != IO_WQ_CANCEL_NOTFOUND)
    ret = true;


    \
     
     \ /
      Last update: 2021-02-11 16:39    [W:4.100 / U:0.112 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site