From: Harman Kalra <hkalra@marvell.com>
To: Anatoly Burakov <anatoly.burakov@intel.com>
Cc: <dev@dpdk.org>, David Hunt <david.hunt@intel.com>,
<liang.j.ma@intel.com>, <reshma.pattan@intel.com>
Subject: Re: [dpdk-dev] [PATCH 3/3] l3fwd-power: add interrupt-only mode
Date: Fri, 29 May 2020 18:49:56 +0530 [thread overview]
Message-ID: <20200529131955.GA83122@outlook.office365.com> (raw)
In-Reply-To: <ceb97619dcaa91188757e6cf330870a6dfe97de6.1590656906.git.anatoly.burakov@intel.com>
On Thu, May 28, 2020 at 10:13:51AM +0100, Anatoly Burakov wrote:
> In addition to existing modes, add a mode which is very similar to
> legacy mode, but does not do frequency scaling, and thus does not
> depend on the power library.
>
> Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>
> ---
> examples/l3fwd-power/main.c | 215 +++++++++++++++++++++++++++++++++---
> 1 file changed, 202 insertions(+), 13 deletions(-)
>
> diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
> index 5cee9d5387..4161e01974 100644
> --- a/examples/l3fwd-power/main.c
> +++ b/examples/l3fwd-power/main.c
> @@ -195,9 +195,11 @@ static int parse_ptype; /**< Parse packet type using rx callback, and */
> /**< disabled by default */
>
> enum appmode {
> - APP_MODE_LEGACY = 0,
> + APP_MODE_DEFAULT = 0,
> + APP_MODE_LEGACY,
> APP_MODE_EMPTY_POLL,
> - APP_MODE_TELEMETRY
> + APP_MODE_TELEMETRY,
> + APP_MODE_INTERRUPT
> };
>
> enum appmode app_mode;
> @@ -900,6 +902,170 @@ static int event_register(struct lcore_conf *qconf)
>
> return 0;
> }
> +
> +/* main processing loop */
> +static int main_intr_loop(__rte_unused void *dummy)
> +{
> + struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
> + unsigned int lcore_id;
> + uint64_t prev_tsc, diff_tsc, cur_tsc;
> + int i, j, nb_rx;
> + uint8_t queueid;
> + uint16_t portid;
> + struct lcore_conf *qconf;
> + struct lcore_rx_queue *rx_queue;
> + uint32_t lcore_rx_idle_count = 0;
> + uint32_t lcore_idle_hint = 0;
> + int intr_en = 0;
> +
> + const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /
> + US_PER_S * BURST_TX_DRAIN_US;
> +
> + prev_tsc = 0;
> +
> + lcore_id = rte_lcore_id();
> + qconf = &lcore_conf[lcore_id];
> +
> + if (qconf->n_rx_queue == 0) {
> + RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n",
> + lcore_id);
> + return 0;
> + }
> +
> + RTE_LOG(INFO, L3FWD_POWER, "entering main interrupt loop on lcore %u\n",
> + lcore_id);
> +
> + for (i = 0; i < qconf->n_rx_queue; i++) {
> + portid = qconf->rx_queue_list[i].port_id;
> + queueid = qconf->rx_queue_list[i].queue_id;
> + RTE_LOG(INFO, L3FWD_POWER,
> + " -- lcoreid=%u portid=%u rxqueueid=%hhu\n",
> + lcore_id, portid, queueid);
> + }
> +
> + /* add into event wait list */
> + if (event_register(qconf) == 0)
> + intr_en = 1;
> + else
> + RTE_LOG(INFO, L3FWD_POWER, "RX interrupt won't enable.\n");
> +
> + while (!is_done()) {
> + stats[lcore_id].nb_iteration_looped++;
> +
> + cur_tsc = rte_rdtsc();
> +
> + /*
> + * TX burst queue drain
> + */
> + diff_tsc = cur_tsc - prev_tsc;
> + if (unlikely(diff_tsc > drain_tsc)) {
> + for (i = 0; i < qconf->n_tx_port; ++i) {
> + portid = qconf->tx_port_id[i];
> + rte_eth_tx_buffer_flush(portid,
> + qconf->tx_queue_id[portid],
> + qconf->tx_buffer[portid]);
> + }
> + prev_tsc = cur_tsc;
> + }
> +
> +start_rx:
> + /*
> + * Read packet from RX queues
> + */
> + lcore_rx_idle_count = 0;
> + for (i = 0; i < qconf->n_rx_queue; ++i) {
> + rx_queue = &(qconf->rx_queue_list[i]);
> + rx_queue->idle_hint = 0;
> + portid = rx_queue->port_id;
> + queueid = rx_queue->queue_id;
> +
> + nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst,
> + MAX_PKT_BURST);
> +
> + stats[lcore_id].nb_rx_processed += nb_rx;
> + if (unlikely(nb_rx == 0)) {
> + /**
> + * no packet received from rx queue, try to
> + * sleep for a while forcing CPU enter deeper
> + * C states.
> + */
> + rx_queue->zero_rx_packet_count++;
> +
> + if (rx_queue->zero_rx_packet_count <=
> + MIN_ZERO_POLL_COUNT)
> + continue;
> +
> + rx_queue->idle_hint = power_idle_heuristic(
> + rx_queue->zero_rx_packet_count);
> + lcore_rx_idle_count++;
> + } else {
> + rx_queue->zero_rx_packet_count = 0;
> + }
> +
> + /* Prefetch first packets */
> + for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) {
> + rte_prefetch0(rte_pktmbuf_mtod(
> + pkts_burst[j], void *));
> + }
> +
> + /* Prefetch and forward already prefetched packets */
> + for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) {
> + rte_prefetch0(rte_pktmbuf_mtod(
> + pkts_burst[j + PREFETCH_OFFSET],
> + void *));
> + l3fwd_simple_forward(
> + pkts_burst[j], portid, qconf);
> + }
> +
> + /* Forward remaining prefetched packets */
> + for (; j < nb_rx; j++) {
> + l3fwd_simple_forward(
> + pkts_burst[j], portid, qconf);
> + }
> + }
> +
> + if (unlikely(lcore_rx_idle_count == qconf->n_rx_queue)) {
> + /**
> + * All Rx queues empty in recent consecutive polls,
> + * sleep in a conservative manner, meaning sleep as
> + * less as possible.
> + */
> + for (i = 1,
> + lcore_idle_hint = qconf->rx_queue_list[0].idle_hint;
> + i < qconf->n_rx_queue; ++i) {
> + rx_queue = &(qconf->rx_queue_list[i]);
> + if (rx_queue->idle_hint < lcore_idle_hint)
> + lcore_idle_hint = rx_queue->idle_hint;
> + }
> +
> + if (lcore_idle_hint < SUSPEND_THRESHOLD)
> + /**
> + * execute "pause" instruction to avoid context
> + * switch which generally take hundred of
> + * microseconds for short sleep.
> + */
> + rte_delay_us(lcore_idle_hint);
> + else {
> + /* suspend until rx interrupt triggers */
> + if (intr_en) {
> + turn_on_off_intr(qconf, 1);
> + sleep_until_rx_interrupt(
> + qconf->n_rx_queue);
> + turn_on_off_intr(qconf, 0);
> + /**
> + * start receiving packets immediately
> + */
> + if (likely(!is_done()))
> + goto start_rx;
> + }
> + }
> + stats[lcore_id].sleep_time += lcore_idle_hint;
> + }
> + }
> +
> + return 0;
> +}
> +
> /* main processing loop */
> static int
> main_telemetry_loop(__rte_unused void *dummy)
> @@ -1126,7 +1292,7 @@ main_empty_poll_loop(__rte_unused void *dummy)
> }
> /* main processing loop */
> static int
> -main_loop(__rte_unused void *dummy)
> +main_legacy_loop(__rte_unused void *dummy)
> {
> struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
> unsigned lcore_id;
> @@ -1438,7 +1604,8 @@ print_usage(const char *prgname)
> " --empty-poll: enable empty poll detection"
> " follow (training_flag, high_threshold, med_threshold)\n"
> " --telemetry: enable telemetry mode, to update"
> - " empty polls, full polls, and core busyness to telemetry\n",
> + " empty polls, full polls, and core busyness to telemetry\n"
> + " --interrupt-only: enable interrupt-only mode\n",
> prgname);
> }
>
> @@ -1582,6 +1749,7 @@ parse_ep_config(const char *q_arg)
> }
> #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype"
> #define CMD_LINE_OPT_TELEMETRY "telemetry"
> +#define CMD_LINE_OPT_INTERRUPT_ONLY "interrupt-only"
>
> /* Parse the argument given in the command line of the application */
> static int
> @@ -1601,6 +1769,7 @@ parse_args(int argc, char **argv)
> {"empty-poll", 1, 0, 0},
> {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0},
> {CMD_LINE_OPT_TELEMETRY, 0, 0, 0},
> + {CMD_LINE_OPT_INTERRUPT_ONLY, 0, 0, 0},
> {NULL, 0, 0, 0}
> };
>
> @@ -1674,8 +1843,8 @@ parse_args(int argc, char **argv)
>
> if (!strncmp(lgopts[option_index].name,
> "empty-poll", 10)) {
> - if (app_mode == APP_MODE_TELEMETRY) {
> - printf(" empty-poll cannot be enabled as telemetry mode is enabled\n");
> + if (app_mode != APP_MODE_DEFAULT) {
> + printf(" empty-poll mode is mutually exclusive with other modes\n");
> return -1;
> }
> app_mode = APP_MODE_EMPTY_POLL;
> @@ -1692,14 +1861,25 @@ parse_args(int argc, char **argv)
> if (!strncmp(lgopts[option_index].name,
> CMD_LINE_OPT_TELEMETRY,
> sizeof(CMD_LINE_OPT_TELEMETRY))) {
> - if (app_mode == APP_MODE_EMPTY_POLL) {
> - printf("telemetry mode cannot be enabled as empty poll mode is enabled\n");
> + if (app_mode != APP_MODE_DEFAULT) {
> + printf(" telemetry mode is mutually exclusive with other modes\n");
> return -1;
> }
> app_mode = APP_MODE_TELEMETRY;
> printf("telemetry mode is enabled\n");
> }
>
> + if (!strncmp(lgopts[option_index].name,
> + CMD_LINE_OPT_INTERRUPT_ONLY,
> + sizeof(CMD_LINE_OPT_INTERRUPT_ONLY))) {
> + if (app_mode != APP_MODE_DEFAULT) {
> + printf(" interrupt-only mode is mutually exclusive with other modes\n");
> + return -1;
> + }
> + app_mode = APP_MODE_INTERRUPT;
> + printf("interrupt-only mode is enabled\n");
> + }
> +
> if (!strncmp(lgopts[option_index].name,
> "enable-jumbo", 12)) {
> struct option lenopts =
> @@ -2253,7 +2433,12 @@ main(int argc, char **argv)
> if (ret < 0)
> rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
>
> - if (app_mode != APP_MODE_TELEMETRY && init_power_library())
> + if (app_mode == APP_MODE_DEFAULT)
> + app_mode = APP_MODE_LEGACY;
> +
> + /* only legacy and empty poll mode rely on power library */
> + if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
> + init_power_library())
> rte_exit(EXIT_FAILURE, "init_power_library failed\n");
>
Hi,
Rather than just exiting from here can we have a else condition to
automatically enter into the "interrupt only" mode.
Please correct me if I am missing something.
Thanks
Harman
> if (update_lcore_params() < 0)
> @@ -2277,7 +2462,8 @@ main(int argc, char **argv)
> RTE_ETH_FOREACH_DEV(portid) {
> struct rte_eth_conf local_port_conf = port_conf;
> /* not all app modes need interrupts */
> - bool need_intr = app_mode == APP_MODE_LEGACY;
> + bool need_intr = app_mode == APP_MODE_LEGACY ||
> + app_mode == APP_MODE_INTERRUPT;
>
> /* skip ports that are not enabled */
> if ((enabled_port_mask & (1 << portid)) == 0) {
> @@ -2526,12 +2712,12 @@ main(int argc, char **argv)
>
> /* launch per-lcore init on every lcore */
> if (app_mode == APP_MODE_LEGACY) {
> - rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER);
> + rte_eal_mp_remote_launch(main_legacy_loop, NULL, CALL_MASTER);
> } else if (app_mode == APP_MODE_EMPTY_POLL) {
> empty_poll_stop = false;
> rte_eal_mp_remote_launch(main_empty_poll_loop, NULL,
> SKIP_MASTER);
> - } else {
> + } else if (app_mode == APP_MODE_TELEMETRY) {
> unsigned int i;
>
> /* Init metrics library */
> @@ -2555,6 +2741,8 @@ main(int argc, char **argv)
> "Returns global power stats. Parameters: None");
> rte_eal_mp_remote_launch(main_telemetry_loop, NULL,
> SKIP_MASTER);
> + } else if (app_mode == APP_MODE_INTERRUPT) {
> + rte_eal_mp_remote_launch(main_intr_loop, NULL, CALL_MASTER);
> }
>
> if (app_mode == APP_MODE_EMPTY_POLL || app_mode == APP_MODE_TELEMETRY)
> @@ -2577,7 +2765,8 @@ main(int argc, char **argv)
> if (app_mode == APP_MODE_EMPTY_POLL)
> rte_power_empty_poll_stat_free();
>
> - if (app_mode != APP_MODE_TELEMETRY && deinit_power_library())
> + if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
> + deinit_power_library())
> rte_exit(EXIT_FAILURE, "deinit_power_library failed\n");
>
> if (rte_eal_cleanup() < 0)
> --
> 2.17.1
next prev parent reply other threads:[~2020-05-29 13:20 UTC|newest]
Thread overview: 39+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-05-28 9:13 [dpdk-dev] [PATCH 0/3] Add interrupt-only mode to l3fwd-power Anatoly Burakov
2020-05-28 9:13 ` [dpdk-dev] [PATCH 1/3] l3fwd-power: disable interrupts by default Anatoly Burakov
2020-05-28 9:13 ` [dpdk-dev] [PATCH 2/3] l3fwd-power: only allow supported power library envs Anatoly Burakov
2020-05-28 9:13 ` [dpdk-dev] [PATCH 3/3] l3fwd-power: add interrupt-only mode Anatoly Burakov
2020-05-29 13:19 ` Harman Kalra [this message]
2020-05-29 14:19 ` Burakov, Anatoly
2020-05-30 10:02 ` [dpdk-dev] [EXT] " Harman Kalra
2020-06-01 12:50 ` Burakov, Anatoly
2020-06-02 10:23 ` Harman Kalra
2020-06-02 12:16 ` Harman Kalra
2020-06-15 11:31 ` Burakov, Anatoly
2020-06-15 11:43 ` Jerin Jacob
2020-06-15 15:05 ` Burakov, Anatoly
2020-06-15 15:21 ` Jerin Jacob
2020-06-15 15:45 ` Burakov, Anatoly
2020-06-15 16:29 ` Jerin Jacob
2020-06-16 9:31 ` Burakov, Anatoly
2020-06-16 17:09 ` Jerin Jacob
2020-06-08 1:24 ` [dpdk-dev] [PATCH 0/3] Add interrupt-only mode to l3fwd-power Wang, Yinan
2020-06-18 17:18 ` [dpdk-dev] [PATCH v2 0/7] " Anatoly Burakov
2020-06-19 10:53 ` [dpdk-dev] [PATCH v3 " Anatoly Burakov
2020-07-11 11:35 ` Thomas Monjalon
2020-06-19 10:53 ` [dpdk-dev] [PATCH v3 1/7] l3fwd-power: disable interrupts by default Anatoly Burakov
2020-06-19 10:53 ` [dpdk-dev] [PATCH v3 2/7] l3fwd-power: only allow supported power library envs Anatoly Burakov
2020-06-19 10:53 ` [dpdk-dev] [PATCH v3 3/7] l3fwd-power: code style and flow fixes Anatoly Burakov
2020-06-19 10:53 ` [dpdk-dev] [PATCH v3 4/7] l3fwd-power: add support for requesting legacy mode Anatoly Burakov
2020-06-19 10:53 ` [dpdk-dev] [PATCH v3 5/7] l3fwd-power: add interrupt-only mode Anatoly Burakov
2020-06-19 10:53 ` [dpdk-dev] [PATCH v3 6/7] power: add API to probe support for a specific env Anatoly Burakov
2020-06-19 10:53 ` [dpdk-dev] [PATCH v3 7/7] l3fwd-power: add auto-selection of default mode Anatoly Burakov
2020-07-11 10:07 ` Thomas Monjalon
2020-06-18 17:18 ` [dpdk-dev] [PATCH v2 1/7] l3fwd-power: disable interrupts by default Anatoly Burakov
2020-06-18 17:18 ` [dpdk-dev] [PATCH v2 2/7] l3fwd-power: only allow supported power library envs Anatoly Burakov
2020-06-18 17:18 ` [dpdk-dev] [PATCH v2 3/7] l3fwd-power: code style and flow fixes Anatoly Burakov
2020-06-18 17:18 ` [dpdk-dev] [PATCH v2 4/7] l3fwd-power: add support for requesting legacy mode Anatoly Burakov
2020-06-18 17:18 ` [dpdk-dev] [PATCH v2 5/7] l3fwd-power: add interrupt-only mode Anatoly Burakov
2020-06-18 17:18 ` [dpdk-dev] [PATCH v2 6/7] power: add API to probe support for a specific env Anatoly Burakov
2020-06-18 17:18 ` [dpdk-dev] [PATCH v2 7/7] l3fwd-power: add auto-selection of default mode Anatoly Burakov
2020-06-19 7:37 ` [dpdk-dev] [EXT] " Harman Kalra
2020-06-19 9:56 ` Burakov, Anatoly
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200529131955.GA83122@outlook.office365.com \
--to=hkalra@marvell.com \
--cc=anatoly.burakov@intel.com \
--cc=david.hunt@intel.com \
--cc=dev@dpdk.org \
--cc=liang.j.ma@intel.com \
--cc=reshma.pattan@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).