From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from EX13-EDG-OU-001.vmware.com (ex13-edg-ou-001.vmware.com [208.91.0.189]) by dpdk.org (Postfix) with ESMTP id 6D19EFB01 for ; Mon, 6 Mar 2017 18:18:03 +0100 (CET) Received: from sc9-mailhost3.vmware.com (10.113.161.73) by EX13-EDG-OU-001.vmware.com (10.113.208.155) with Microsoft SMTP Server id 15.0.1156.6; Mon, 6 Mar 2017 09:17:15 -0800 Received: from shri-linux.eng.vmware.com (shri-linux.eng.vmware.com [10.33.72.16]) by sc9-mailhost3.vmware.com (Postfix) with ESMTP id 18A08404A8; Mon, 6 Mar 2017 09:18:02 -0800 (PST) From: Shrikrishna Khare To: CC: , Shrikrishna Khare , Guolin Yang Date: Mon, 6 Mar 2017 09:17:45 -0800 Message-ID: <1488820666-24445-7-git-send-email-skhare@vmware.com> X-Mailer: git-send-email 2.6.2 In-Reply-To: <1488820666-24445-1-git-send-email-skhare@vmware.com> References: <1488820666-24445-1-git-send-email-skhare@vmware.com> MIME-Version: 1.0 Content-Type: text/plain Received-SPF: None (EX13-EDG-OU-001.vmware.com: skhare@vmware.com does not designate permitted sender hosts) Subject: [dpdk-dev] [PATCH v2 6/7] net/vmxnet3: add cmd to register memory region X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 06 Mar 2017 17:18:03 -0000 In vmxnet3 version 3, the emulation added support for the vmxnet3 driver to communicate information about the memory regions the driver will use for rx/tx buffers. The driver can also indicate which rx/tx queue the memory region is applicable for. If this information is communicated to the emulation, the emulation will always keep these memory regions mapped, thereby avoiding the mapping/unmapping overhead for every packet. Signed-off-by: Shrikrishna Khare Signed-off-by: Guolin Yang Acked-by: Yong Wang Acked-by: Jin Heo --- drivers/net/vmxnet3/base/vmxnet3_defs.h | 25 ++++++++ drivers/net/vmxnet3/vmxnet3_ethdev.c | 102 ++++++++++++++++++++++++++++++++ drivers/net/vmxnet3/vmxnet3_ethdev.h | 2 + 3 files changed, 129 insertions(+) diff --git a/drivers/net/vmxnet3/base/vmxnet3_defs.h b/drivers/net/vmxnet3/base/vmxnet3_defs.h index c708498..bfa9622 100644 --- a/drivers/net/vmxnet3/base/vmxnet3_defs.h +++ b/drivers/net/vmxnet3/base/vmxnet3_defs.h @@ -111,6 +111,7 @@ typedef enum { VMXNET3_CMD_ACTIVATE_VF, VMXNET3_CMD_RESERVED3, VMXNET3_CMD_RESERVED4, + VMXNET3_CMD_REGISTER_MEMREGS, VMXNET3_CMD_FIRST_GET = 0xF00D0000, VMXNET3_CMD_GET_QUEUE_STATUS = VMXNET3_CMD_FIRST_GET, @@ -722,6 +723,30 @@ struct Vmxnet3_SetPolling { #include "vmware_pack_end.h" Vmxnet3_SetPolling; +typedef +#include "vmware_pack_begin.h" +struct Vmxnet3_MemoryRegion { + __le64 startPA; + __le32 length; + __le16 txQueueBits; /* bit n corresponding to tx queue n */ + __le16 rxQueueBits; /* bit n corresponding to rx queue n */ +} +#include "vmware_pack_end.h" +Vmxnet3_MemoryRegion; + +#define MAX_MEMORY_REGION_PER_QUEUE 16 +#define MAX_MEMORY_REGION_PER_DEVICE 256 + +typedef +#include "vmware_pack_begin.h" +struct Vmxnet3_MemRegs { + __le16 numRegs; + __le16 pad[3]; + Vmxnet3_MemoryRegion memRegs[1]; +} +#include "vmware_pack_end.h" +Vmxnet3_MemRegs; + /* * If the command data <= 16 bytes, use the shared memory direcly. * Otherwise, use the variable length configuration descriptor. diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.c b/drivers/net/vmxnet3/vmxnet3_ethdev.c index 8591ce1..96f1585 100644 --- a/drivers/net/vmxnet3/vmxnet3_ethdev.c +++ b/drivers/net/vmxnet3/vmxnet3_ethdev.c @@ -489,6 +489,92 @@ vmxnet3_write_mac(struct vmxnet3_hw *hw, const uint8_t *addr) } static int +vmxnet3_dev_setup_memreg(struct rte_eth_dev *dev) +{ + struct vmxnet3_hw *hw = dev->data->dev_private; + Vmxnet3_DriverShared *shared = hw->shared; + Vmxnet3_CmdInfo *cmdInfo; + struct rte_mempool *mp[VMXNET3_MAX_RX_QUEUES]; + uint8_t index[VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES]; + uint32_t num, i, j, size; + + if (hw->memRegsPA == 0) { + const struct rte_memzone *mz; + + size = sizeof(Vmxnet3_MemRegs) + + (VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES) * + sizeof(Vmxnet3_MemoryRegion); + + mz = gpa_zone_reserve(dev, size, "memRegs", rte_socket_id(), 8, + 1); + if (mz == NULL) { + PMD_INIT_LOG(ERR, "ERROR: Creating memRegs zone"); + return -ENOMEM; + } + memset(mz->addr, 0, mz->len); + hw->memRegs = mz->addr; + hw->memRegsPA = mz->phys_addr; + } + + num = hw->num_rx_queues; + + for (i = 0; i < num; i++) { + vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i]; + + mp[i] = rxq->mp; + index[i] = 1 << i; + } + + /* + * The same mempool could be used by multiple queues. In such a case, + * remove duplicate mempool entries. Only one entry is kept with + * bitmask indicating queues that are using this mempool. + */ + for (i = 1; i < num; i++) { + for (j = 0; j < i; j++) { + if (mp[i] == mp[j]) { + mp[i] = NULL; + index[j] |= 1 << i; + break; + } + } + } + + j = 0; + for (i = 0; i < num; i++) { + if (mp[i] == NULL) + continue; + + Vmxnet3_MemoryRegion *mr = &hw->memRegs->memRegs[j]; + + mr->startPA = + (uintptr_t)STAILQ_FIRST(&mp[i]->mem_list)->phys_addr; + mr->length = STAILQ_FIRST(&mp[i]->mem_list)->len <= INT32_MAX ? + STAILQ_FIRST(&mp[i]->mem_list)->len : INT32_MAX; + mr->txQueueBits = index[i]; + mr->rxQueueBits = index[i]; + + PMD_INIT_LOG(INFO, + "index: %u startPA: %" PRIu64 " length: %u, " + "rxBits: %x", + j, mr->startPA, mr->length, mr->rxQueueBits); + j++; + } + hw->memRegs->numRegs = j; + PMD_INIT_LOG(INFO, "numRegs: %u", j); + + size = sizeof(Vmxnet3_MemRegs) + + (j - 1) * sizeof(Vmxnet3_MemoryRegion); + + cmdInfo = &shared->cu.cmdInfo; + cmdInfo->varConf.confVer = 1; + cmdInfo->varConf.confLen = size; + cmdInfo->varConf.confPA = hw->memRegsPA; + + return 0; +} + +static int vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) { struct rte_eth_conf port_conf = dev->data->dev_conf; @@ -628,6 +714,20 @@ vmxnet3_dev_start(struct rte_eth_dev *dev) return -EINVAL; } + /* Setup memory region for rx buffers */ + ret = vmxnet3_dev_setup_memreg(dev); + if (ret == 0) { + VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, + VMXNET3_CMD_REGISTER_MEMREGS); + ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); + if (ret != 0) + PMD_INIT_LOG(DEBUG, + "Failed in setup memory region cmd\n"); + ret = 0; + } else { + PMD_INIT_LOG(DEBUG, "Failed to setup memory region\n"); + } + /* Disable interrupts */ vmxnet3_disable_intr(hw); @@ -641,6 +741,8 @@ vmxnet3_dev_start(struct rte_eth_dev *dev) return ret; } + hw->adapter_stopped = FALSE; + /* Setting proper Rx Mode and issue Rx Mode Update command */ vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_UCAST | VMXNET3_RXM_BCAST, 1); diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.h b/drivers/net/vmxnet3/vmxnet3_ethdev.h index 9d00cd1..7a03262 100644 --- a/drivers/net/vmxnet3/vmxnet3_ethdev.h +++ b/drivers/net/vmxnet3/vmxnet3_ethdev.h @@ -119,6 +119,8 @@ struct vmxnet3_hw { uint64_t rss_confPA; vmxnet3_mf_table_t *mf_table; uint32_t shadow_vfta[VMXNET3_VFT_SIZE]; + Vmxnet3_MemRegs *memRegs; + uint64_t memRegsPA; #define VMXNET3_VFT_TABLE_SIZE (VMXNET3_VFT_SIZE * sizeof(uint32_t)) }; -- 2.6.2