lkml.org 
[lkml]   [2015]   [May]   [27]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH 3.13.y-ckt 25/95] iscsi-target: Convert iscsi_thread_set usage to kthread.h
    Date
    3.13.11-ckt21 -stable review patch.  If anyone has any objections, please let me know.

    ------------------

    From: Nicholas Bellinger <nab@linux-iscsi.org>

    commit 88dcd2dab5c23b1c9cfc396246d8f476c872f0ca upstream.

    This patch converts iscsi-target code to use modern kthread.h API
    callers for creating RX/TX threads for each new iscsi_conn descriptor,
    and releasing associated RX/TX threads during connection shutdown.

    This is done using iscsit_start_kthreads() -> kthread_run() to start
    new kthreads from within iscsi_post_login_handler(), and invoking
    kthread_stop() from existing iscsit_close_connection() code.

    Also, convert iscsit_logout_post_handler_closesession() code to use
    cmpxchg when determing when iscsit_cause_connection_reinstatement()
    needs to sleep waiting for completion.

    Reported-by: Sagi Grimberg <sagig@mellanox.com>
    Tested-by: Sagi Grimberg <sagig@mellanox.com>
    Cc: Slava Shwartsman <valyushash@gmail.com>
    Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
    [ luis: backported to 3.16:
    - file rename: include/target/iscsi/iscsi_target_core.h ->
    drivers/target/iscsi/iscsi_target_core.h
    - adjusted context ]
    Signed-off-by: Luis Henriques <luis.henriques@canonical.com>
    Signed-off-by: Kamal Mostafa <kamal@canonical.com>
    ---
    drivers/target/iscsi/iscsi_target.c | 104 +++++++++++++-----------------
    drivers/target/iscsi/iscsi_target_core.h | 7 ++
    drivers/target/iscsi/iscsi_target_erl0.c | 13 ++--
    drivers/target/iscsi/iscsi_target_login.c | 59 +++++++++++++++--
    4 files changed, 114 insertions(+), 69 deletions(-)

    diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c
    index 505aa4e..8da53d7 100644
    --- a/drivers/target/iscsi/iscsi_target.c
    +++ b/drivers/target/iscsi/iscsi_target.c
    @@ -517,7 +517,7 @@ static struct iscsit_transport iscsi_target_transport = {

    static int __init iscsi_target_init_module(void)
    {
    - int ret = 0;
    + int ret = 0, size;

    pr_debug("iSCSI-Target "ISCSIT_VERSION"\n");

    @@ -526,6 +526,7 @@ static int __init iscsi_target_init_module(void)
    pr_err("Unable to allocate memory for iscsit_global\n");
    return -1;
    }
    + spin_lock_init(&iscsit_global->ts_bitmap_lock);
    mutex_init(&auth_id_lock);
    spin_lock_init(&sess_idr_lock);
    idr_init(&tiqn_idr);
    @@ -535,15 +536,11 @@ static int __init iscsi_target_init_module(void)
    if (ret < 0)
    goto out;

    - ret = iscsi_thread_set_init();
    - if (ret < 0)
    + size = BITS_TO_LONGS(ISCSIT_BITMAP_BITS) * sizeof(long);
    + iscsit_global->ts_bitmap = vzalloc(size);
    + if (!iscsit_global->ts_bitmap) {
    + pr_err("Unable to allocate iscsit_global->ts_bitmap\n");
    goto configfs_out;
    -
    - if (iscsi_allocate_thread_sets(TARGET_THREAD_SET_COUNT) !=
    - TARGET_THREAD_SET_COUNT) {
    - pr_err("iscsi_allocate_thread_sets() returned"
    - " unexpected value!\n");
    - goto ts_out1;
    }

    lio_qr_cache = kmem_cache_create("lio_qr_cache",
    @@ -552,7 +549,7 @@ static int __init iscsi_target_init_module(void)
    if (!lio_qr_cache) {
    pr_err("nable to kmem_cache_create() for"
    " lio_qr_cache\n");
    - goto ts_out2;
    + goto bitmap_out;
    }

    lio_dr_cache = kmem_cache_create("lio_dr_cache",
    @@ -596,10 +593,8 @@ dr_out:
    kmem_cache_destroy(lio_dr_cache);
    qr_out:
    kmem_cache_destroy(lio_qr_cache);
    -ts_out2:
    - iscsi_deallocate_thread_sets();
    -ts_out1:
    - iscsi_thread_set_free();
    +bitmap_out:
    + vfree(iscsit_global->ts_bitmap);
    configfs_out:
    iscsi_target_deregister_configfs();
    out:
    @@ -609,8 +604,6 @@ out:

    static void __exit iscsi_target_cleanup_module(void)
    {
    - iscsi_deallocate_thread_sets();
    - iscsi_thread_set_free();
    iscsit_release_discovery_tpg();
    iscsit_unregister_transport(&iscsi_target_transport);
    kmem_cache_destroy(lio_qr_cache);
    @@ -620,6 +613,7 @@ static void __exit iscsi_target_cleanup_module(void)

    iscsi_target_deregister_configfs();

    + vfree(iscsit_global->ts_bitmap);
    kfree(iscsit_global);
    }

    @@ -3652,17 +3646,16 @@ static int iscsit_send_reject(

    void iscsit_thread_get_cpumask(struct iscsi_conn *conn)
    {
    - struct iscsi_thread_set *ts = conn->thread_set;
    int ord, cpu;
    /*
    - * thread_id is assigned from iscsit_global->ts_bitmap from
    - * within iscsi_thread_set.c:iscsi_allocate_thread_sets()
    + * bitmap_id is assigned from iscsit_global->ts_bitmap from
    + * within iscsit_start_kthreads()
    *
    - * Here we use thread_id to determine which CPU that this
    - * iSCSI connection's iscsi_thread_set will be scheduled to
    + * Here we use bitmap_id to determine which CPU that this
    + * iSCSI connection's RX/TX threads will be scheduled to
    * execute upon.
    */
    - ord = ts->thread_id % cpumask_weight(cpu_online_mask);
    + ord = conn->bitmap_id % cpumask_weight(cpu_online_mask);
    for_each_online_cpu(cpu) {
    if (ord-- == 0) {
    cpumask_set_cpu(cpu, conn->conn_cpumask);
    @@ -3854,7 +3847,7 @@ check_rsp_state:
    switch (state) {
    case ISTATE_SEND_LOGOUTRSP:
    if (!iscsit_logout_post_handler(cmd, conn))
    - goto restart;
    + return -ECONNRESET;
    /* fall through */
    case ISTATE_SEND_STATUS:
    case ISTATE_SEND_ASYNCMSG:
    @@ -3882,8 +3875,6 @@ check_rsp_state:

    err:
    return -1;
    -restart:
    - return -EAGAIN;
    }

    static int iscsit_handle_response_queue(struct iscsi_conn *conn)
    @@ -3910,21 +3901,13 @@ static int iscsit_handle_response_queue(struct iscsi_conn *conn)
    int iscsi_target_tx_thread(void *arg)
    {
    int ret = 0;
    - struct iscsi_conn *conn;
    - struct iscsi_thread_set *ts = arg;
    + struct iscsi_conn *conn = arg;
    /*
    * Allow ourselves to be interrupted by SIGINT so that a
    * connection recovery / failure event can be triggered externally.
    */
    allow_signal(SIGINT);

    -restart:
    - conn = iscsi_tx_thread_pre_handler(ts);
    - if (!conn)
    - goto out;
    -
    - ret = 0;
    -
    while (!kthread_should_stop()) {
    /*
    * Ensure that both TX and RX per connection kthreads
    @@ -3933,11 +3916,9 @@ restart:
    iscsit_thread_check_cpumask(conn, current, 1);

    wait_event_interruptible(conn->queues_wq,
    - !iscsit_conn_all_queues_empty(conn) ||
    - ts->status == ISCSI_THREAD_SET_RESET);
    + !iscsit_conn_all_queues_empty(conn));

    - if ((ts->status == ISCSI_THREAD_SET_RESET) ||
    - signal_pending(current))
    + if (signal_pending(current))
    goto transport_err;

    get_immediate:
    @@ -3948,15 +3929,14 @@ get_immediate:
    ret = iscsit_handle_response_queue(conn);
    if (ret == 1)
    goto get_immediate;
    - else if (ret == -EAGAIN)
    - goto restart;
    + else if (ret == -ECONNRESET)
    + goto out;
    else if (ret < 0)
    goto transport_err;
    }

    transport_err:
    iscsit_take_action_for_connection_exit(conn);
    - goto restart;
    out:
    return 0;
    }
    @@ -4045,8 +4025,7 @@ int iscsi_target_rx_thread(void *arg)
    int ret;
    u8 buffer[ISCSI_HDR_LEN], opcode;
    u32 checksum = 0, digest = 0;
    - struct iscsi_conn *conn = NULL;
    - struct iscsi_thread_set *ts = arg;
    + struct iscsi_conn *conn = arg;
    struct kvec iov;
    /*
    * Allow ourselves to be interrupted by SIGINT so that a
    @@ -4054,11 +4033,6 @@ int iscsi_target_rx_thread(void *arg)
    */
    allow_signal(SIGINT);

    -restart:
    - conn = iscsi_rx_thread_pre_handler(ts);
    - if (!conn)
    - goto out;
    -
    if (conn->conn_transport->transport_type == ISCSI_INFINIBAND) {
    struct completion comp;
    int rc;
    @@ -4068,7 +4042,7 @@ restart:
    if (rc < 0)
    goto transport_err;

    - goto out;
    + goto transport_err;
    }

    while (!kthread_should_stop()) {
    @@ -4144,8 +4118,6 @@ transport_err:
    if (!signal_pending(current))
    atomic_set(&conn->transport_failed, 1);
    iscsit_take_action_for_connection_exit(conn);
    - goto restart;
    -out:
    return 0;
    }

    @@ -4207,7 +4179,24 @@ int iscsit_close_connection(
    if (conn->conn_transport->transport_type == ISCSI_TCP)
    complete(&conn->conn_logout_comp);

    - iscsi_release_thread_set(conn);
    + if (!strcmp(current->comm, ISCSI_RX_THREAD_NAME)) {
    + if (conn->tx_thread &&
    + cmpxchg(&conn->tx_thread_active, true, false)) {
    + send_sig(SIGINT, conn->tx_thread, 1);
    + kthread_stop(conn->tx_thread);
    + }
    + } else if (!strcmp(current->comm, ISCSI_TX_THREAD_NAME)) {
    + if (conn->rx_thread &&
    + cmpxchg(&conn->rx_thread_active, true, false)) {
    + send_sig(SIGINT, conn->rx_thread, 1);
    + kthread_stop(conn->rx_thread);
    + }
    + }
    +
    + spin_lock(&iscsit_global->ts_bitmap_lock);
    + bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id,
    + get_order(1));
    + spin_unlock(&iscsit_global->ts_bitmap_lock);

    iscsit_stop_timers_for_cmds(conn);
    iscsit_stop_nopin_response_timer(conn);
    @@ -4486,15 +4475,13 @@ static void iscsit_logout_post_handler_closesession(
    struct iscsi_conn *conn)
    {
    struct iscsi_session *sess = conn->sess;
    -
    - iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD);
    - iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD);
    + int sleep = cmpxchg(&conn->tx_thread_active, true, false);

    atomic_set(&conn->conn_logout_remove, 0);
    complete(&conn->conn_logout_comp);

    iscsit_dec_conn_usage_count(conn);
    - iscsit_stop_session(sess, 1, 1);
    + iscsit_stop_session(sess, sleep, sleep);
    iscsit_dec_session_usage_count(sess);
    target_put_session(sess->se_sess);
    }
    @@ -4502,13 +4489,12 @@ static void iscsit_logout_post_handler_closesession(
    static void iscsit_logout_post_handler_samecid(
    struct iscsi_conn *conn)
    {
    - iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD);
    - iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD);
    + int sleep = cmpxchg(&conn->tx_thread_active, true, false);

    atomic_set(&conn->conn_logout_remove, 0);
    complete(&conn->conn_logout_comp);

    - iscsit_cause_connection_reinstatement(conn, 1);
    + iscsit_cause_connection_reinstatement(conn, sleep);
    iscsit_dec_conn_usage_count(conn);
    }

    diff --git a/drivers/target/iscsi/iscsi_target_core.h b/drivers/target/iscsi/iscsi_target_core.h
    index 48f7b3b..31a68f5 100644
    --- a/drivers/target/iscsi/iscsi_target_core.h
    +++ b/drivers/target/iscsi/iscsi_target_core.h
    @@ -601,6 +601,11 @@ struct iscsi_conn {
    struct iscsi_session *sess;
    /* Pointer to thread_set in use for this conn's threads */
    struct iscsi_thread_set *thread_set;
    + int bitmap_id;
    + int rx_thread_active;
    + struct task_struct *rx_thread;
    + int tx_thread_active;
    + struct task_struct *tx_thread;
    /* list_head for session connection list */
    struct list_head conn_list;
    } ____cacheline_aligned;
    @@ -868,10 +873,12 @@ struct iscsit_global {
    /* Unique identifier used for the authentication daemon */
    u32 auth_id;
    u32 inactive_ts;
    +#define ISCSIT_BITMAP_BITS 262144
    /* Thread Set bitmap count */
    int ts_bitmap_count;
    /* Thread Set bitmap pointer */
    unsigned long *ts_bitmap;
    + spinlock_t ts_bitmap_lock;
    /* Used for iSCSI discovery session authentication */
    struct iscsi_node_acl discovery_acl;
    struct iscsi_portal_group *discovery_tpg;
    diff --git a/drivers/target/iscsi/iscsi_target_erl0.c b/drivers/target/iscsi/iscsi_target_erl0.c
    index 0d1e6ee..7396d90 100644
    --- a/drivers/target/iscsi/iscsi_target_erl0.c
    +++ b/drivers/target/iscsi/iscsi_target_erl0.c
    @@ -864,7 +864,10 @@ void iscsit_connection_reinstatement_rcfr(struct iscsi_conn *conn)
    }
    spin_unlock_bh(&conn->state_lock);

    - iscsi_thread_set_force_reinstatement(conn);
    + if (conn->tx_thread && conn->tx_thread_active)
    + send_sig(SIGINT, conn->tx_thread, 1);
    + if (conn->rx_thread && conn->rx_thread_active)
    + send_sig(SIGINT, conn->rx_thread, 1);

    sleep:
    wait_for_completion(&conn->conn_wait_rcfr_comp);
    @@ -889,10 +892,10 @@ void iscsit_cause_connection_reinstatement(struct iscsi_conn *conn, int sleep)
    return;
    }

    - if (iscsi_thread_set_force_reinstatement(conn) < 0) {
    - spin_unlock_bh(&conn->state_lock);
    - return;
    - }
    + if (conn->tx_thread && conn->tx_thread_active)
    + send_sig(SIGINT, conn->tx_thread, 1);
    + if (conn->rx_thread && conn->rx_thread_active)
    + send_sig(SIGINT, conn->rx_thread, 1);

    atomic_set(&conn->connection_reinstatement, 1);
    if (!sleep) {
    diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c
    index 439c534..1a8aea1 100644
    --- a/drivers/target/iscsi/iscsi_target_login.c
    +++ b/drivers/target/iscsi/iscsi_target_login.c
    @@ -681,6 +681,51 @@ static void iscsi_post_login_start_timers(struct iscsi_conn *conn)
    iscsit_start_nopin_timer(conn);
    }

    +int iscsit_start_kthreads(struct iscsi_conn *conn)
    +{
    + int ret = 0;
    +
    + spin_lock(&iscsit_global->ts_bitmap_lock);
    + conn->bitmap_id = bitmap_find_free_region(iscsit_global->ts_bitmap,
    + ISCSIT_BITMAP_BITS, get_order(1));
    + spin_unlock(&iscsit_global->ts_bitmap_lock);
    +
    + if (conn->bitmap_id < 0) {
    + pr_err("bitmap_find_free_region() failed for"
    + " iscsit_start_kthreads()\n");
    + return -ENOMEM;
    + }
    +
    + conn->tx_thread = kthread_run(iscsi_target_tx_thread, conn,
    + "%s", ISCSI_TX_THREAD_NAME);
    + if (IS_ERR(conn->tx_thread)) {
    + pr_err("Unable to start iscsi_target_tx_thread\n");
    + ret = PTR_ERR(conn->tx_thread);
    + goto out_bitmap;
    + }
    + conn->tx_thread_active = true;
    +
    + conn->rx_thread = kthread_run(iscsi_target_rx_thread, conn,
    + "%s", ISCSI_RX_THREAD_NAME);
    + if (IS_ERR(conn->rx_thread)) {
    + pr_err("Unable to start iscsi_target_rx_thread\n");
    + ret = PTR_ERR(conn->rx_thread);
    + goto out_tx;
    + }
    + conn->rx_thread_active = true;
    +
    + return 0;
    +out_tx:
    + kthread_stop(conn->tx_thread);
    + conn->tx_thread_active = false;
    +out_bitmap:
    + spin_lock(&iscsit_global->ts_bitmap_lock);
    + bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id,
    + get_order(1));
    + spin_unlock(&iscsit_global->ts_bitmap_lock);
    + return ret;
    +}
    +
    int iscsi_post_login_handler(
    struct iscsi_np *np,
    struct iscsi_conn *conn,
    @@ -691,7 +736,7 @@ int iscsi_post_login_handler(
    struct se_session *se_sess = sess->se_sess;
    struct iscsi_portal_group *tpg = sess->tpg;
    struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
    - struct iscsi_thread_set *ts;
    + int rc;

    iscsit_inc_conn_usage_count(conn);

    @@ -706,7 +751,6 @@ int iscsi_post_login_handler(
    /*
    * SCSI Initiator -> SCSI Target Port Mapping
    */
    - ts = iscsi_get_thread_set();
    if (!zero_tsih) {
    iscsi_set_session_parameters(sess->sess_ops,
    conn->param_list, 0);
    @@ -733,9 +777,11 @@ int iscsi_post_login_handler(
    sess->sess_ops->InitiatorName);
    spin_unlock_bh(&sess->conn_lock);

    - iscsi_post_login_start_timers(conn);
    + rc = iscsit_start_kthreads(conn);
    + if (rc)
    + return rc;

    - iscsi_activate_thread_set(conn, ts);
    + iscsi_post_login_start_timers(conn);
    /*
    * Determine CPU mask to ensure connection's RX and TX kthreads
    * are scheduled on the same CPU.
    @@ -792,8 +838,11 @@ int iscsi_post_login_handler(
    " iSCSI Target Portal Group: %hu\n", tpg->nsessions, tpg->tpgt);
    spin_unlock_bh(&se_tpg->session_lock);

    + rc = iscsit_start_kthreads(conn);
    + if (rc)
    + return rc;
    +
    iscsi_post_login_start_timers(conn);
    - iscsi_activate_thread_set(conn, ts);
    /*
    * Determine CPU mask to ensure connection's RX and TX kthreads
    * are scheduled on the same CPU.
    --
    1.9.1


    \
     
     \ /
      Last update: 2015-05-27 21:41    [W:4.380 / U:1.388 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site