lkml.org 
[lkml]   [2021]   [Jan]   [4]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH v8 18/20] dlb: add dynamic queue map register operations
    Date
    Adds the "dynamic" map procedure and register operations. If a queue map
    is requested after the domain is started, the driver must disable the
    requested queue and wait for it to quiesce before mapping it to the
    requested port.

    Signed-off-by: Gage Eads <gage.eads@intel.com>
    Signed-off-by: Mike Ximing Chen <mike.ximing.chen@intel.com>
    Reviewed-by: Björn Töpel <bjorn.topel@intel.com>
    Reviewed-by: Dan Williams <dan.j.williams@intel.com>
    ---
    drivers/misc/dlb/dlb_resource.c | 388 +++++++++++++++++++++++++++++++-
    1 file changed, 385 insertions(+), 3 deletions(-)

    diff --git a/drivers/misc/dlb/dlb_resource.c b/drivers/misc/dlb/dlb_resource.c
    index 67ecbd4150c5..776285ee92d7 100644
    --- a/drivers/misc/dlb/dlb_resource.c
    +++ b/drivers/misc/dlb/dlb_resource.c
    @@ -337,6 +337,39 @@ dlb_get_domain_dir_pq(u32 id,
    return NULL;
    }

    +static struct dlb_ldb_queue *
    +dlb_get_ldb_queue_from_id(struct dlb_hw *hw,
    + u32 id,
    + bool vdev_req,
    + unsigned int vdev_id)
    +{
    + struct dlb_function_resources *rsrcs;
    + struct dlb_hw_domain *domain;
    + struct dlb_ldb_queue *queue;
    +
    + if (id >= DLB_MAX_NUM_LDB_QUEUES)
    + return NULL;
    +
    + rsrcs = (vdev_req) ? &hw->vdev[vdev_id] : &hw->pf;
    +
    + if (!vdev_req)
    + return &hw->rsrcs.ldb_queues[id];
    +
    + list_for_each_entry(domain, &rsrcs->used_domains, func_list) {
    + list_for_each_entry(queue, &domain->used_ldb_queues, domain_list) {
    + if (queue->id.virt_id == id)
    + return queue;
    + }
    + }
    +
    + list_for_each_entry(queue, &rsrcs->avail_ldb_queues, func_list) {
    + if (queue->id.virt_id == id)
    + return queue;
    + }
    +
    + return NULL;
    +}
    +
    static struct dlb_ldb_queue *
    dlb_get_domain_ldb_queue(u32 id,
    bool vdev_req,
    @@ -2340,6 +2373,76 @@ static void dlb_ldb_port_change_qid_priority(struct dlb_hw *hw,
    port->qid_map[slot].priority = args->priority;
    }

    +static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,
    + struct dlb_ldb_port *port,
    + struct dlb_ldb_queue *queue,
    + int slot)
    +{
    + u32 ctrl = 0;
    + u32 active;
    + u32 enq;
    +
    + /* Set the atomic scheduling haswork bit */
    + active = DLB_CSR_RD(hw, LSP_QID_AQED_ACTIVE_CNT(queue->id.phys_id));
    +
    + BITS_SET(ctrl, port->id.phys_id, LSP_LDB_SCHED_CTRL_CQ);
    + BITS_SET(ctrl, slot, LSP_LDB_SCHED_CTRL_QIDIX);
    + BIT_SET(ctrl, LSP_LDB_SCHED_CTRL_VALUE);
    + BITS_SET(ctrl, (u32)(BITS_GET(active, LSP_QID_AQED_ACTIVE_CNT_COUNT) > 0),
    + LSP_LDB_SCHED_CTRL_RLIST_HASWORK_V);
    +
    + /* Set the non-atomic scheduling haswork bit */
    + DLB_CSR_WR(hw, LSP_LDB_SCHED_CTRL, ctrl);
    +
    + enq = DLB_CSR_RD(hw,
    + LSP_QID_LDB_ENQUEUE_CNT(queue->id.phys_id));
    +
    + memset(&ctrl, 0, sizeof(ctrl));
    +
    + BITS_SET(ctrl, port->id.phys_id, LSP_LDB_SCHED_CTRL_CQ);
    + BITS_SET(ctrl, slot, LSP_LDB_SCHED_CTRL_QIDIX);
    + BIT_SET(ctrl, LSP_LDB_SCHED_CTRL_VALUE);
    + BITS_SET(ctrl, (u32)(BITS_GET(enq, LSP_QID_LDB_ENQUEUE_CNT_COUNT) > 0),
    + LSP_LDB_SCHED_CTRL_NALB_HASWORK_V);
    +
    + DLB_CSR_WR(hw, LSP_LDB_SCHED_CTRL, ctrl);
    +
    + dlb_flush_csr(hw);
    +
    + return 0;
    +}
    +
    +static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,
    + struct dlb_ldb_port *port,
    + int slot)
    +{
    + u32 ctrl = 0;
    +
    + BITS_SET(ctrl, port->id.phys_id, LSP_LDB_SCHED_CTRL_CQ);
    + BITS_SET(ctrl, slot, LSP_LDB_SCHED_CTRL_QIDIX);
    + BIT_SET(ctrl, LSP_LDB_SCHED_CTRL_INFLIGHT_OK_V);
    +
    + DLB_CSR_WR(hw, LSP_LDB_SCHED_CTRL, ctrl);
    +
    + dlb_flush_csr(hw);
    +}
    +
    +static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,
    + struct dlb_ldb_port *port,
    + int slot)
    +{
    + u32 ctrl = 0;
    +
    + BITS_SET(ctrl, port->id.phys_id, LSP_LDB_SCHED_CTRL_CQ);
    + BITS_SET(ctrl, slot, LSP_LDB_SCHED_CTRL_QIDIX);
    + BIT_SET(ctrl, LSP_LDB_SCHED_CTRL_VALUE);
    + BIT_SET(ctrl, LSP_LDB_SCHED_CTRL_INFLIGHT_OK_V);
    +
    + DLB_CSR_WR(hw, LSP_LDB_SCHED_CTRL, ctrl);
    +
    + dlb_flush_csr(hw);
    +}
    +
    static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
    struct dlb_ldb_queue *queue)
    {
    @@ -2350,13 +2453,224 @@ static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
    DLB_CSR_WR(hw, LSP_QID_LDB_INFL_LIM(queue->id.phys_id), infl_lim);
    }

    +static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,
    + struct dlb_ldb_queue *queue)
    +{
    + DLB_CSR_WR(hw,
    + LSP_QID_LDB_INFL_LIM(queue->id.phys_id),
    + LSP_QID_LDB_INFL_LIM_RST);
    +}
    +
    +/*
    + * dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as
    + * their function names imply, and should only be called by the dynamic CQ
    + * mapping code.
    + */
    +static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,
    + struct dlb_hw_domain *domain,
    + struct dlb_ldb_queue *queue)
    +{
    + struct dlb_ldb_port *port;
    + int slot, i;
    +
    + for (i = 0; i < DLB_NUM_COS_DOMAINS; i++) {
    + list_for_each_entry(port, &domain->used_ldb_ports[i], domain_list) {
    + enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
    +
    + if (!dlb_port_find_slot_queue(port, state,
    + queue, &slot))
    + continue;
    +
    + if (port->enabled)
    + dlb_ldb_port_cq_disable(hw, port);
    + }
    + }
    +}
    +
    +static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,
    + struct dlb_hw_domain *domain,
    + struct dlb_ldb_queue *queue)
    +{
    + struct dlb_ldb_port *port;
    + int slot, i;
    +
    + for (i = 0; i < DLB_NUM_COS_DOMAINS; i++) {
    + list_for_each_entry(port, &domain->used_ldb_ports[i], domain_list) {
    + enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
    +
    + if (!dlb_port_find_slot_queue(port, state,
    + queue, &slot))
    + continue;
    +
    + if (port->enabled)
    + dlb_ldb_port_cq_enable(hw, port);
    + }
    + }
    +}
    +
    +static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,
    + struct dlb_hw_domain *domain,
    + struct dlb_ldb_port *port,
    + struct dlb_ldb_queue *queue)
    +{
    + enum dlb_qid_map_state state;
    + int slot, ret, i;
    + u32 infl_cnt;
    + u8 prio;
    +
    + infl_cnt = DLB_CSR_RD(hw, LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
    +
    + if (BITS_GET(infl_cnt, LSP_QID_LDB_INFL_CNT_COUNT)) {
    + DLB_HW_ERR(hw,
    + "[%s()] Internal error: non-zero QID inflight count\n",
    + __func__);
    + return -EINVAL;
    + }
    +
    + /*
    + * Static map the port and set its corresponding has_work bits.
    + */
    + state = DLB_QUEUE_MAP_IN_PROG;
    + if (!dlb_port_find_slot_queue(port, state, queue, &slot))
    + return -EINVAL;
    +
    + prio = port->qid_map[slot].priority;
    +
    + /*
    + * Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
    + * the port's qid_map state.
    + */
    + ret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);
    + if (ret)
    + return ret;
    +
    + ret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);
    + if (ret)
    + return ret;
    +
    + /*
    + * Ensure IF_status(cq,qid) is 0 before enabling the port to
    + * prevent spurious schedules to cause the queue's inflight
    + * count to increase.
    + */
    + dlb_ldb_port_clear_queue_if_status(hw, port, slot);
    +
    + /* Reset the queue's inflight status */
    + for (i = 0; i < DLB_NUM_COS_DOMAINS; i++) {
    + list_for_each_entry(port, &domain->used_ldb_ports[i], domain_list) {
    + state = DLB_QUEUE_MAPPED;
    + if (!dlb_port_find_slot_queue(port, state,
    + queue, &slot))
    + continue;
    +
    + dlb_ldb_port_set_queue_if_status(hw, port, slot);
    + }
    + }
    +
    + dlb_ldb_queue_set_inflight_limit(hw, queue);
    +
    + /* Re-enable CQs mapped to this queue */
    + dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
    +
    + /* If this queue has other mappings pending, clear its inflight limit */
    + if (queue->num_pending_additions > 0)
    + dlb_ldb_queue_clear_inflight_limit(hw, queue);
    +
    + return 0;
    +}
    +
    +/**
    + * dlb_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
    + * @hw: dlb_hw handle for a particular device.
    + * @port: load-balanced port
    + * @queue: load-balanced queue
    + * @priority: queue servicing priority
    + *
    + * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
    + * at a later point, and <0 if an error occurred.
    + */
    +static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,
    + struct dlb_ldb_port *port,
    + struct dlb_ldb_queue *queue,
    + u8 priority)
    +{
    + enum dlb_qid_map_state state;
    + struct dlb_hw_domain *domain;
    + int domain_id, slot, ret;
    + u32 infl_cnt;
    +
    + domain_id = port->domain_id.phys_id;
    +
    + domain = dlb_get_domain_from_id(hw, domain_id, false, 0);
    + if (!domain) {
    + DLB_HW_ERR(hw,
    + "[%s()] Internal error: unable to find domain %d\n",
    + __func__, port->domain_id.phys_id);
    + return -EINVAL;
    + }
    +
    + /*
    + * Set the QID inflight limit to 0 to prevent further scheduling of the
    + * queue.
    + */
    + DLB_CSR_WR(hw, LSP_QID_LDB_INFL_LIM(queue->id.phys_id), 0);
    +
    + if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {
    + DLB_HW_ERR(hw,
    + "Internal error: No available unmapped slots\n");
    + return -EFAULT;
    + }
    +
    + port->qid_map[slot].qid = queue->id.phys_id;
    + port->qid_map[slot].priority = priority;
    +
    + state = DLB_QUEUE_MAP_IN_PROG;
    + ret = dlb_port_slot_state_transition(hw, port, queue, slot, state);
    + if (ret)
    + return ret;
    +
    + infl_cnt = DLB_CSR_RD(hw, LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
    +
    + if (BITS_GET(infl_cnt, LSP_QID_LDB_INFL_CNT_COUNT))
    + return 1;
    +
    + /*
    + * Disable the affected CQ, and the CQs already mapped to the QID,
    + * before reading the QID's inflight count a second time. There is an
    + * unlikely race in which the QID may schedule one more QE after we
    + * read an inflight count of 0, and disabling the CQs guarantees that
    + * the race will not occur after a re-read of the inflight count
    + * register.
    + */
    + if (port->enabled)
    + dlb_ldb_port_cq_disable(hw, port);
    +
    + dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
    +
    + infl_cnt = DLB_CSR_RD(hw, LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
    +
    + if (BITS_GET(infl_cnt, LSP_QID_LDB_INFL_CNT_COUNT)) {
    + if (port->enabled)
    + dlb_ldb_port_cq_enable(hw, port);
    +
    + dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
    +
    + return 1;
    + }
    +
    + return dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
    +}
    +
    static int dlb_ldb_port_map_qid(struct dlb_hw *hw,
    struct dlb_hw_domain *domain,
    struct dlb_ldb_port *port,
    struct dlb_ldb_queue *queue,
    u8 prio)
    {
    - return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
    + if (domain->started)
    + return dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);
    + else
    + return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
    }

    static void
    @@ -2890,12 +3204,80 @@ dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,
    return 0;
    }

    +static void dlb_domain_finish_map_port(struct dlb_hw *hw,
    + struct dlb_hw_domain *domain,
    + struct dlb_ldb_port *port)
    +{
    + int i;
    +
    + for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
    + struct dlb_ldb_queue *queue;
    + u32 infl_cnt;
    + int qid;
    +
    + if (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROG)
    + continue;
    +
    + qid = port->qid_map[i].qid;
    +
    + queue = dlb_get_ldb_queue_from_id(hw, qid, false, 0);
    +
    + if (!queue) {
    + DLB_HW_ERR(hw,
    + "[%s()] Internal error: unable to find queue %d\n",
    + __func__, qid);
    + continue;
    + }
    +
    + infl_cnt = DLB_CSR_RD(hw, LSP_QID_LDB_INFL_CNT(qid));
    +
    + if (BITS_GET(infl_cnt, LSP_QID_LDB_INFL_CNT_COUNT))
    + continue;
    +
    + /*
    + * Disable the affected CQ, and the CQs already mapped to the
    + * QID, before reading the QID's inflight count a second time.
    + * There is an unlikely race in which the QID may schedule one
    + * more QE after we read an inflight count of 0, and disabling
    + * the CQs guarantees that the race will not occur after a
    + * re-read of the inflight count register.
    + */
    + if (port->enabled)
    + dlb_ldb_port_cq_disable(hw, port);
    +
    + dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
    +
    + infl_cnt = DLB_CSR_RD(hw, LSP_QID_LDB_INFL_CNT(qid));
    +
    + if (BITS_GET(infl_cnt, LSP_QID_LDB_INFL_CNT_COUNT)) {
    + if (port->enabled)
    + dlb_ldb_port_cq_enable(hw, port);
    +
    + dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
    +
    + continue;
    + }
    +
    + dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
    + }
    +}
    +
    static unsigned int
    dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,
    struct dlb_hw_domain *domain)
    {
    - /* Placeholder */
    - return 0;
    + struct dlb_ldb_port *port;
    + int i;
    +
    + if (!domain->configured || domain->num_pending_additions == 0)
    + return 0;
    +
    + for (i = 0; i < DLB_NUM_COS_DOMAINS; i++) {
    + list_for_each_entry(port, &domain->used_ldb_ports[i], domain_list)
    + dlb_domain_finish_map_port(hw, domain, port);
    + }
    +
    + return domain->num_pending_additions;
    }

    static void dlb_log_map_qid(struct dlb_hw *hw,
    --
    2.17.1
    \
     
     \ /
      Last update: 2021-01-05 04:01    [W:4.122 / U:0.008 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site