lkml.org 
[lkml]   [2011]   [Apr]   [11]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    From
    SubjectRe: Strange block/scsi/workqueue issue
    Hello,

    (cc'ing James. The original message is http://lkml.org/lkml/2011/4/11/175 )

    Please read from the bottom up.

    On Mon, Apr 11, 2011 at 03:56:03PM +0100, Steven Whitehouse wrote:
    > [<ffffffff8167b8e5>] schedule_timeout+0x295/0x310
    > [<ffffffff8167a650>] wait_for_common+0x120/0x170
    > [<ffffffff8167a748>] wait_for_completion+0x18/0x20
    > [<ffffffff810aba4c>] wait_on_cpu_work+0xec/0x100
    > [<ffffffff810abb3b>] wait_on_work+0xdb/0x150
    > [<ffffffff810abc33>] __cancel_work_timer+0x83/0x130
    > [<ffffffff810abced>] cancel_delayed_work_sync+0xd/0x10

    4. which in turn tries to sync cancel q->delay_work. Oops, deadlock.

    > [<ffffffff813b24b4>] blk_sync_queue+0x24/0x50

    3. and calls into blk_sync_queue()

    > [<ffffffff813b24ef>] blk_cleanup_queue+0xf/0x60
    > [<ffffffff81479a89>] scsi_free_queue+0x9/0x10
    > [<ffffffff8147d30b>] scsi_device_dev_release_usercontext+0xeb/0x140
    > [<ffffffff810ac826>] execute_in_process_context+0x86/0xa0

    2. It triggers SCSI device release

    > [<ffffffff8147d1f7>] scsi_device_dev_release+0x17/0x20
    > [<ffffffff814609f2>] device_release+0x22/0x90
    > [<ffffffff813c8165>] kobject_release+0x45/0x90
    > [<ffffffff813c9767>] kref_put+0x37/0x70
    > [<ffffffff813c8027>] kobject_put+0x27/0x60
    > [<ffffffff81460822>] put_device+0x12/0x20
    > [<ffffffff81478bd9>] scsi_request_fn+0xb9/0x4a0
    > [<ffffffff813aff2a>] __blk_run_queue+0x6a/0x110
    > [<ffffffff813b1f66>] blk_delay_work+0x26/0x40

    1. Workqueue starting execution of q->delay_work and scsi_request_fn()
    is run from there.

    > [<ffffffff810aa9c7>] process_one_work+0x197/0x520
    > [<ffffffff810acfec>] worker_thread+0x15c/0x330
    > [<ffffffff810b1f16>] kthread+0xa6/0xb0
    > [<ffffffff81687064>] kernel_thread_helper+0x4/0x10

    So, q->delay_work ends up waiting for itself. I'd like to blame SCSI
    (as it also fits my agenda to kill execute_in_process_context ;-) for
    diving all the way into blk_cleanup_queue() directly from request_fn.

    Does the following patch fix the problem?

    Thanks.

    Subject: scsi: don't use execute_in_process_context()

    SCSI is the only subsystem which uses execute_in_process_context() and
    its use is racy against module unload. ie. the reap work is not
    properly flushed and could still be running after the scsi module is
    unloaded.

    Although execute_in_process_context() can be more efficient when the
    caller already has a context, in this case, the call paths are quite
    cold and the difference is practically meaningless. With commit
    c8efcc25 (workqueue: allow chained queueing during destruction), the
    race condition can easily be fixed by using a dedicated workqueue and
    destroying it on module unload.

    Create and use scsi_wq instead of execute_in_process_context().

    * scsi_device->ew is replaced with release_work. scsi_target->ew is
    replaced with reap_work.

    * Both works are initialized with the respective release/reap function
    during device/target init. scsi_target_reap_usercontext() is moved
    upwards to avoid needing forward declaration.

    * scsi_alloc_target() now explicitly flushes the reap_work of the
    found dying target before putting it instead of depending on
    flush_scheduled_work().

    For more info on the issues, please read the following thread.

    http://thread.gmane.org/gmane.linux.scsi/62923

    Signed-off-by: Tejun Heo <tj@kernel.org>
    ---
    drivers/scsi/scsi.c | 15 +++++++++++++--
    drivers/scsi/scsi_scan.c | 26 +++++++++++++-------------
    drivers/scsi/scsi_sysfs.c | 8 +++++---
    include/scsi/scsi_device.h | 6 ++++--
    4 files changed, 35 insertions(+), 20 deletions(-)

    Index: work/drivers/scsi/scsi_scan.c
    ===================================================================
    --- work.orig/drivers/scsi/scsi_scan.c
    +++ work/drivers/scsi/scsi_scan.c
    @@ -362,6 +362,16 @@ int scsi_is_target_device(const struct d
    }
    EXPORT_SYMBOL(scsi_is_target_device);

    +static void scsi_target_reap_usercontext(struct work_struct *work)
    +{
    + struct scsi_target *starget =
    + container_of(work, struct scsi_target, reap_work);
    +
    + transport_remove_device(&starget->dev);
    + device_del(&starget->dev);
    + scsi_target_destroy(starget);
    +}
    +
    static struct scsi_target *__scsi_find_target(struct device *parent,
    int channel, uint id)
    {
    @@ -427,6 +437,7 @@ static struct scsi_target *scsi_alloc_ta
    starget->state = STARGET_CREATED;
    starget->scsi_level = SCSI_2;
    starget->max_target_blocked = SCSI_DEFAULT_TARGET_BLOCKED;
    + INIT_WORK(&starget->reap_work, scsi_target_reap_usercontext);
    retry:
    spin_lock_irqsave(shost->host_lock, flags);

    @@ -462,21 +473,11 @@ static struct scsi_target *scsi_alloc_ta
    }
    /* Unfortunately, we found a dying target; need to
    * wait until it's dead before we can get a new one */
    + flush_work(&found_target->reap_work);
    put_device(&found_target->dev);
    - flush_scheduled_work();
    goto retry;
    }

    -static void scsi_target_reap_usercontext(struct work_struct *work)
    -{
    - struct scsi_target *starget =
    - container_of(work, struct scsi_target, ew.work);
    -
    - transport_remove_device(&starget->dev);
    - device_del(&starget->dev);
    - scsi_target_destroy(starget);
    -}
    -
    /**
    * scsi_target_reap - check to see if target is in use and destroy if not
    * @starget: target to be checked
    @@ -507,8 +508,7 @@ void scsi_target_reap(struct scsi_target
    if (state == STARGET_CREATED)
    scsi_target_destroy(starget);
    else
    - execute_in_process_context(scsi_target_reap_usercontext,
    - &starget->ew);
    + queue_work(scsi_wq, &starget->reap_work);
    }

    /**
    Index: work/drivers/scsi/scsi_sysfs.c
    ===================================================================
    --- work.orig/drivers/scsi/scsi_sysfs.c
    +++ work/drivers/scsi/scsi_sysfs.c
    @@ -300,7 +300,7 @@ static void scsi_device_dev_release_user
    struct list_head *this, *tmp;
    unsigned long flags;

    - sdev = container_of(work, struct scsi_device, ew.work);
    + sdev = container_of(work, struct scsi_device, release_work);

    parent = sdev->sdev_gendev.parent;
    starget = to_scsi_target(parent);
    @@ -343,8 +343,8 @@ static void scsi_device_dev_release_user
    static void scsi_device_dev_release(struct device *dev)
    {
    struct scsi_device *sdp = to_scsi_device(dev);
    - execute_in_process_context(scsi_device_dev_release_usercontext,
    - &sdp->ew);
    +
    + queue_work(scsi_wq, &sdp->release_work);
    }

    static struct class sdev_class = {
    @@ -1069,6 +1069,8 @@ void scsi_sysfs_device_initialize(struct
    dev_set_name(&sdev->sdev_dev, "%d:%d:%d:%d",
    sdev->host->host_no, sdev->channel, sdev->id, sdev->lun);
    sdev->scsi_level = starget->scsi_level;
    + INIT_WORK(&sdev->release_work, scsi_device_dev_release_usercontext);
    +
    transport_setup_device(&sdev->sdev_gendev);
    spin_lock_irqsave(shost->host_lock, flags);
    list_add_tail(&sdev->same_target_siblings, &starget->devices);
    Index: work/include/scsi/scsi_device.h
    ===================================================================
    --- work.orig/include/scsi/scsi_device.h
    +++ work/include/scsi/scsi_device.h
    @@ -168,7 +168,7 @@ struct scsi_device {
    struct device sdev_gendev,
    sdev_dev;

    - struct execute_work ew; /* used to get process context on put */
    + struct work_struct release_work; /* for process context on put */

    struct scsi_dh_data *scsi_dh_data;
    enum scsi_device_state sdev_state;
    @@ -259,7 +259,7 @@ struct scsi_target {
    #define SCSI_DEFAULT_TARGET_BLOCKED 3

    char scsi_level;
    - struct execute_work ew;
    + struct work_struct reap_work;
    enum scsi_target_state state;
    void *hostdata; /* available to low-level driver */
    unsigned long starget_data[0]; /* for the transport */
    @@ -277,6 +277,8 @@ static inline struct scsi_target *scsi_t
    #define starget_printk(prefix, starget, fmt, a...) \
    dev_printk(prefix, &(starget)->dev, fmt, ##a)

    +extern struct workqueue_struct *scsi_wq;
    +
    extern struct scsi_device *__scsi_add_device(struct Scsi_Host *,
    uint, uint, uint, void *hostdata);
    extern int scsi_add_device(struct Scsi_Host *host, uint channel,
    Index: work/drivers/scsi/scsi.c
    ===================================================================
    --- work.orig/drivers/scsi/scsi.c
    +++ work/drivers/scsi/scsi.c
    @@ -70,6 +70,11 @@
    #define CREATE_TRACE_POINTS
    #include <trace/events/scsi.h>

    +/*
    + * Utility multithreaded workqueue for SCSI.
    + */
    +struct workqueue_struct *scsi_wq;
    +
    static void scsi_done(struct scsi_cmnd *cmd);

    /*
    @@ -1306,11 +1311,14 @@ MODULE_PARM_DESC(scsi_logging_level, "a

    static int __init init_scsi(void)
    {
    - int error;
    + int error = -ENOMEM;

    + scsi_wq = alloc_workqueue("scsi", 0, 0);
    + if (!scsi_wq)
    + return error;
    error = scsi_init_queue();
    if (error)
    - return error;
    + goto cleanup_wq;
    error = scsi_init_procfs();
    if (error)
    goto cleanup_queue;
    @@ -1342,6 +1350,8 @@ cleanup_procfs:
    scsi_exit_procfs();
    cleanup_queue:
    scsi_exit_queue();
    +cleanup_wq:
    + destroy_workqueue(scsi_wq);
    printk(KERN_ERR "SCSI subsystem failed to initialize, error = %d\n",
    -error);
    return error;
    @@ -1356,6 +1366,7 @@ static void __exit exit_scsi(void)
    scsi_exit_devinfo();
    scsi_exit_procfs();
    scsi_exit_queue();
    + destroy_workqueue(scsi_wq);
    }

    subsys_initcall(init_scsi);

    \
     
     \ /
      Last update: 2011-04-11 19:21    [W:0.052 / U:1.456 seconds]
    ©2003-2016 Jasper Spaans. hosted at Digital OceanAdvertise on this site