[dpdk-dev] [RFC PATCH 0/2] using scapy to generate packet templates

[dpdk-dev] [RFC PATCH 0/2] using scapy to generate packet templates

[Xueming Li]

The target is to make testpmd flexible to generate all kinds of packet
in just one line. Examples:
 
# send UDP packet to port 0 queue 0
testpmd> scapy 0 Ether()/IP()/UDP()/"hello_world"

# send 4 L3 VXLAN packets with inner UDP dport 1-4 to port 1 queue 1-4, one packet per queue
testpmd> scapy 1 Ether()/IP()/UDP()/VXLAN()/IPv6()/UDP(dport=(1,4))/"cool"

# Random
testpmd> scapy 2 Ether(src=RandMAC())/IP(dst=RandIP())/TCP(sport=RandShort(),dport=0x1234)

This patch introduce scapy - a python tool to generate static packet templates.
Txonly forwarding mode is modified to support template if available.
 
Two new configuration:
CONFIG_RTE_TEST_PMD_SCAPY=y
CONFIG_RTE_PYTHON=python2.7

Scapy quick demo: http://www.secdev.org/projects/scapy/demo.html

There are 2 other potential solution to support more fancy sending in the future:
1. Invoke scapy and expose mbuf hugepage pointers to python
   This should allow more flexible and dynamic processing
2. Scapy uses wrapped DPDK port as regular port
   Make scapy capable to send high speed streams.

Xueming Li (2):
  app/testpmd: add packet template
  app/testpmd: add scapy command as pkt template

 app/test-pmd/Makefile  |  5 +++
 app/test-pmd/cmdline.c | 99 +++++++++++++++++++++++++++++++++++++++++++++++++-
 app/test-pmd/testpmd.c |  3 ++
 app/test-pmd/testpmd.h |  1 +
 app/test-pmd/txonly.c  | 42 +++++++++++++--------
 config/common_base     |  2 +
 6 files changed, 135 insertions(+), 17 deletions(-)

-- 
2.13.3

[dpdk-dev] [RFC PATCH 1/2] app/testpmd: add packet template

[Xueming Li]

Txonly forwarding mode will check templates, if avaialbe, use templates
for each queue.

Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 app/test-pmd/testpmd.c |  3 +++
 app/test-pmd/testpmd.h |  1 +
 app/test-pmd/txonly.c  | 42 ++++++++++++++++++++++++++----------------
 3 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index a4d4a866b..dacb97888 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -128,6 +128,9 @@ uint8_t mp_anon = 0;
 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
 portid_t nb_peer_eth_addrs = 0;
 
+/* Template of packet to tx */
+struct rte_mbuf *pkt_templ[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];
+
 /*
  * Probed Target Environment.
  */
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 265b75f9f..4090d1e87 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -480,6 +480,7 @@ extern struct fwd_stream **fwd_streams;
 
 extern portid_t nb_peer_eth_addrs; /**< Number of peer ethernet addresses. */
 extern struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
+extern struct rte_mbuf *pkt_templ[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];
 
 extern uint32_t burst_tx_delay_time; /**< Burst tx delay time(us) for mac-retry. */
 extern uint32_t burst_tx_retry_num;  /**< Burst tx retry number for mac-retry. */
diff --git a/app/test-pmd/txonly.c b/app/test-pmd/txonly.c
index 7070ddc3b..e2b189c60 100644
--- a/app/test-pmd/txonly.c
+++ b/app/test-pmd/txonly.c
@@ -186,6 +186,7 @@ pkt_burst_transmit(struct fwd_stream *fs)
 	struct rte_port *txp;
 	struct rte_mbuf *pkt;
 	struct rte_mbuf *pkt_seg;
+	struct rte_mbuf *tmpl;
 	struct rte_mempool *mbp;
 	struct ether_hdr eth_hdr;
 	uint16_t nb_tx;
@@ -206,6 +207,7 @@ pkt_burst_transmit(struct fwd_stream *fs)
 #endif
 
 	mbp = current_fwd_lcore()->mbp;
+	tmpl = pkt_templ[fs->tx_port][fs->tx_queue];
 	txp = &ports[fs->tx_port];
 	vlan_tci = txp->tx_vlan_id;
 	vlan_tci_outer = txp->tx_vlan_id_outer;
@@ -231,7 +233,7 @@ pkt_burst_transmit(struct fwd_stream *fs)
 		 * reset to default value.
 		 */
 		rte_pktmbuf_reset_headroom(pkt);
-		pkt->data_len = tx_pkt_seg_lengths[0];
+		pkt->data_len = tmpl ? tmpl->data_len : tx_pkt_seg_lengths[0];
 		pkt_seg = pkt;
 		if (tx_pkt_split == TX_PKT_SPLIT_RND)
 			nb_segs = random() % tx_pkt_nb_segs + 1;
@@ -251,22 +253,30 @@ pkt_burst_transmit(struct fwd_stream *fs)
 		}
 		pkt_seg->next = NULL; /* Last segment of packet. */
 
-		/*
-		 * Initialize Ethernet header.
-		 */
-		ether_addr_copy(&peer_eth_addrs[fs->peer_addr],&eth_hdr.d_addr);
-		ether_addr_copy(&ports[fs->tx_port].eth_addr, &eth_hdr.s_addr);
-		eth_hdr.ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4);
+		if (tmpl) {
+			copy_buf_to_pkt(
+				rte_pktmbuf_mtod_offset(tmpl, void *, 0),
+				tmpl->data_len, pkt, 0);
+		} else {
+			/*
+			 * Initialize Ethernet header.
+			 */
+			ether_addr_copy(&peer_eth_addrs[fs->peer_addr],
+					&eth_hdr.d_addr);
+			ether_addr_copy(&ports[fs->tx_port].eth_addr,
+					&eth_hdr.s_addr);
+			eth_hdr.ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4);
 
-		/*
-		 * Copy headers in first packet segment(s).
-		 */
-		copy_buf_to_pkt(&eth_hdr, sizeof(eth_hdr), pkt, 0);
-		copy_buf_to_pkt(&pkt_ip_hdr, sizeof(pkt_ip_hdr), pkt,
-				sizeof(struct ether_hdr));
-		copy_buf_to_pkt(&pkt_udp_hdr, sizeof(pkt_udp_hdr), pkt,
-				sizeof(struct ether_hdr) +
-				sizeof(struct ipv4_hdr));
+			/*
+			 * Copy headers in first packet segment(s).
+			 */
+			copy_buf_to_pkt(&eth_hdr, sizeof(eth_hdr), pkt, 0);
+			copy_buf_to_pkt(&pkt_ip_hdr, sizeof(pkt_ip_hdr), pkt,
+					sizeof(struct ether_hdr));
+			copy_buf_to_pkt(&pkt_udp_hdr, sizeof(pkt_udp_hdr), pkt,
+					sizeof(struct ether_hdr) +
+					sizeof(struct ipv4_hdr));
+		}
 
 		/*
 		 * Complete first mbuf of packet and append it to the
-- 
2.13.3

[dpdk-dev] [RFC v1 00/11] scapy/python extension

[Xueming Li]

Thiss patch introduces a set of new elements into DPDK:
1. python/scapy as a standard dpdk library, disabled by default.
   Could be used by any application to integrate powerful python features.
2. tx, rx and expect, blocking mode testpmd CLI command
   tx command send packets with scapy syntax template, support range/enum, be able to generate multiple flows in one command and DPDK speed.
   rx command receive packets with timeout, with several choices on how to dump packets and mbuf headers.
   expect command try to send out packet, receive and compare with the packet sent, support mbuf header and packet header comparation, show detail info or hexdiff if anything different. Expect command is suitable for unit test and regression test by saving batch expect scripts into a file and use 'load' command to invoke them together.
3. pktgen - new testpmd forwarding engine
   Used to support tx, rx and expect blocking command.
   Also support rxonly/loopback/forward/macswap non-blocking idle modes with several packet dump choice.
4. py command - new testpmd CLI
   Run and evaluate python clause in a global context, samples:
    - py a = 123; b=0xaaa; hex(a+b)
    - py eth=Ether();ip=IP();
    - py hexdump(eth/ip/UDP());
    - tx 0 eth/ip
   py shell: sneak into python shell

The purpose of this scapy/python integration is to help programmers to speed up unit test w/o writing complex c code, verify features with batch expect scripts. Testpmd already a widely used platform for people to setup PMD and verify features with rich CLI command, that's why I choose to make new CLI on this familier tool instead of creating a new one.

I made hundreds of expect test cases when developing rte_flow vxlan and GRE features, that makes me pretty confident to touch any code, hope this tool help you as well.

quick guide document:
	https://github.com/steevenlee/dpdk/blob/master_scapy/doc/guides/howto/scapy.rst
github branch:
        https://github.com/steevenlee/dpdk/tree/master_scapy


v1:
* make scapy standard lib, default to disabled
* introduce pktgen forwarding engine for tx, rx and expect command
* py command to evaluate python clause or invoke python shell in testpmd
* batch expect sample file

v0:
* basic scapy lib
* scapy command

Xueming Li (11):
  lib/cmdline: support backspace key
  lib/cmdline: init parse result memeory
  lib/cmdline: add echo support in batch loading from file
  app/testpmd: support command echo in CLI batch loading
  test: update batch loading test
  lib/python: add embedded python lib
  app/testpmd: add python command
  app/testpmd: add pktgen forwarding engine
  app/testpmd: add pktgen engine scapy commands
  test/expect: add expect test scripts
  doc/scapy: add scapy how-to guide

 app/test-pmd/Makefile                    |    6 +
 app/test-pmd/cmdline.c                   |   80 ++-
 app/test-pmd/pktgen.c                    | 1092 ++++++++++++++++++++++++++++++
 app/test-pmd/testpmd.c                   |    1 +
 app/test-pmd/testpmd.h                   |    5 +
 config/common_base                       |    6 +
 doc/guides/howto/scapy.rst               |  300 ++++++++
 lib/Makefile                             |    2 +
 lib/librte_cmdline/cmdline_parse.c       |    2 +
 lib/librte_cmdline/cmdline_rdline.c      |    1 +
 lib/librte_cmdline/cmdline_socket.c      |    5 +-
 lib/librte_cmdline/cmdline_socket.h      |    3 +-
 lib/librte_cmdline/cmdline_vt100.c       |    1 +
 lib/librte_cmdline/cmdline_vt100.h       |    1 +
 lib/librte_eal/common/include/rte_log.h  |    1 +
 lib/librte_python/Makefile               |   60 ++
 lib/librte_python/rte_python.c           |  387 +++++++++++
 lib/librte_python/rte_python.h           |   71 ++
 lib/librte_python/rte_python_version.map |   12 +
 mk/rte.app.mk                            |    1 +
 test/expect/init.exp                     |   28 +
 test/expect/rx.exp                       |  134 ++++
 test/test/test_cmdline_lib.c             |   10 +-
 23 files changed, 2199 insertions(+), 10 deletions(-)
 create mode 100644 app/test-pmd/pktgen.c
 create mode 100644 doc/guides/howto/scapy.rst
 create mode 100644 lib/librte_python/Makefile
 create mode 100644 lib/librte_python/rte_python.c
 create mode 100644 lib/librte_python/rte_python.h
 create mode 100644 lib/librte_python/rte_python_version.map
 create mode 100644 test/expect/init.exp
 create mode 100644 test/expect/rx.exp

-- 
2.13.3

[dpdk-dev] [RFC v1 00/11] scapy/python extension

[Xueming Li]

Thiss patch introduces a set of new elements into DPDK:
1. python/scapy as a standard dpdk library, disabled by default.
   Could be used by any application to integrate powerful python features.
2. tx, rx and expect, blocking mode testpmd CLI command
   tx command send packets with scapy syntax template, support range/enum, be able to generate multiple flows in one command and DPDK speed.
   rx command receive packets with timeout, with several choices on how to dump packets and mbuf headers.
   expect command try to send out packet, receive and compare with the packet sent, support mbuf header and packet header comparation, show detail info or hexdiff if anything different. Expect command is suitable for unit test and regression test by saving batch expect scripts into a file and use 'load' command to invoke them together.
3. pktgen - new testpmd forwarding engine
   Used to support tx, rx and expect blocking command.
   Also support rxonly/loopback/forward/macswap non-blocking idle modes with several packet dump choice.
4. py command - new testpmd CLI
   Run and evaluate python clause in a global context, samples:
    - py a = 123; b=0xaaa; hex(a+b)
    - py eth=Ether();ip=IP();
    - py hexdump(eth/ip/UDP());
    - tx 0 eth/ip
   py shell: sneak into python shell

The purpose of this scapy/python integration is to help programmers to speed up unit test w/o writing complex c code, verify features with batch expect scripts. Testpmd already a widely used platform for people to setup PMD and verify features with rich CLI command, that's why I choose to make new CLI on this familier tool instead of creating a new one.

I made hundreds of expect test cases when developing rte_flow vxlan and GRE features, that makes me pretty confident to touch any code, hope this tool help you as well.

quick guide document:
	https://github.com/steevenlee/dpdk/blob/master_scapy/doc/guides/howto/scapy.rst
github branch:
        https://github.com/steevenlee/dpdk/tree/master_scapy


v1:
* make scapy standard lib, default to disabled
* introduce pktgen forwarding engine for tx, rx and expect command
* py command to evaluate python clause or invoke python shell in testpmd
* batch expect sample file

v0:
* basic scapy lib
* scapy command

Xueming Li (11):
  lib/cmdline: support backspace key
  lib/cmdline: init parse result memeory
  lib/cmdline: add echo support in batch loading from file
  app/testpmd: support command echo in CLI batch loading
  test: update batch loading test
  lib/python: add embedded python lib
  app/testpmd: add python command
  app/testpmd: add pktgen forwarding engine
  app/testpmd: add pktgen engine scapy commands
  test/expect: add expect test scripts
  doc/scapy: add scapy how-to guide

 app/test-pmd/Makefile                    |    6 +
 app/test-pmd/cmdline.c                   |   80 ++-
 app/test-pmd/pktgen.c                    | 1092 ++++++++++++++++++++++++++++++
 app/test-pmd/testpmd.c                   |    1 +
 app/test-pmd/testpmd.h                   |    5 +
 config/common_base                       |    6 +
 doc/guides/howto/scapy.rst               |  300 ++++++++
 lib/Makefile                             |    2 +
 lib/librte_cmdline/cmdline_parse.c       |    2 +
 lib/librte_cmdline/cmdline_rdline.c      |    1 +
 lib/librte_cmdline/cmdline_socket.c      |    5 +-
 lib/librte_cmdline/cmdline_socket.h      |    3 +-
 lib/librte_cmdline/cmdline_vt100.c       |    1 +
 lib/librte_cmdline/cmdline_vt100.h       |    1 +
 lib/librte_eal/common/include/rte_log.h  |    1 +
 lib/librte_python/Makefile               |   60 ++
 lib/librte_python/rte_python.c           |  387 +++++++++++
 lib/librte_python/rte_python.h           |   71 ++
 lib/librte_python/rte_python_version.map |   12 +
 mk/rte.app.mk                            |    1 +
 test/expect/init.exp                     |   28 +
 test/expect/rx.exp                       |  134 ++++
 test/test/test_cmdline_lib.c             |   10 +-
 23 files changed, 2199 insertions(+), 10 deletions(-)
 create mode 100644 app/test-pmd/pktgen.c
 create mode 100644 doc/guides/howto/scapy.rst
 create mode 100644 lib/librte_python/Makefile
 create mode 100644 lib/librte_python/rte_python.c
 create mode 100644 lib/librte_python/rte_python.h
 create mode 100644 lib/librte_python/rte_python_version.map
 create mode 100644 test/expect/init.exp
 create mode 100644 test/expect/rx.exp

-- 
2.13.3

[dpdk-dev] [RFC v1 00/11] scapy/python extension

[Xueming Li]

This patch introduces a set of new elements into DPDK:
1. python/scapy as a standard dpdk library, disabled by default.
   Could be used by any application to integrate powerful python features.
2. tx, rx and expect, blocking mode testpmd CLI command
   tx command send packets with scapy syntax template, support range/enum, be able to generate multiple flows in one command and DPDK speed.
   rx command receive packets with timeout, with several choices on how to dump packets and mbuf headers.
   expect command try to send out packet, receive and compare with the packet sent, support mbuf header and packet header comparation, show detail info or hexdiff if anything different. Expect command is suitable for unit test and regression test by saving batch expect scripts into a file and use 'load' command to invoke them together.
3. pktgen - new testpmd forwarding engine
   Used to support tx, rx and expect blocking command.
   Also support rxonly/loopback/forward/macswap non-blocking idle modes with several packet dump choice.
4. py command - new testpmd CLI
   Run and evaluate python clause in a global context, samples:
    - py a = 123; b=0xaaa; hex(a+b)
    - py eth=Ether();ip=IP();
    - py hexdump(eth/ip/UDP());
    - tx 0 eth/ip
   py shell: sneak into python shell

The purpose of this scapy/python integration is to help programmers to speed up unit test w/o writing complex c code, verify features with batch expect scripts. Testpmd already a widely used platform for people to setup PMD and verify features with rich CLI command, that's why I choose to make new CLI on this familier tool instead of creating a new one.

I made hundreds of expect test cases when developing rte_flow vxlan and GRE features, that makes me pretty confident to touch any code, hope this tool help you as well.

quick guide document:
	https://github.com/steevenlee/dpdk/blob/master_scapy/doc/guides/howto/scapy.rst
github branch:
        https://github.com/steevenlee/dpdk/tree/master_scapy


v1:
* make scapy standard lib, default to disabled
* introduce pktgen forwarding engine for tx, rx and expect command
* py command to evaluate python clause or invoke python shell in testpmd
* batch expect sample file

v0:
* basic scapy lib
* scapy command

Xueming Li (11):
  lib/cmdline: support backspace key
  lib/cmdline: init parse result memeory
  lib/cmdline: add echo support in batch loading from file
  app/testpmd: support command echo in CLI batch loading
  test: update batch loading test
  lib/python: add embedded python lib
  app/testpmd: add python command
  app/testpmd: add pktgen forwarding engine
  app/testpmd: add pktgen engine scapy commands
  test/expect: add expect test scripts
  doc/scapy: add scapy how-to guide

 app/test-pmd/Makefile                    |    6 +
 app/test-pmd/cmdline.c                   |   80 ++-
 app/test-pmd/pktgen.c                    | 1092 ++++++++++++++++++++++++++++++
 app/test-pmd/testpmd.c                   |    1 +
 app/test-pmd/testpmd.h                   |    5 +
 config/common_base                       |    6 +
 doc/guides/howto/scapy.rst               |  300 ++++++++
 lib/Makefile                             |    2 +
 lib/librte_cmdline/cmdline_parse.c       |    2 +
 lib/librte_cmdline/cmdline_rdline.c      |    1 +
 lib/librte_cmdline/cmdline_socket.c      |    5 +-
 lib/librte_cmdline/cmdline_socket.h      |    3 +-
 lib/librte_cmdline/cmdline_vt100.c       |    1 +
 lib/librte_cmdline/cmdline_vt100.h       |    1 +
 lib/librte_eal/common/include/rte_log.h  |    1 +
 lib/librte_python/Makefile               |   60 ++
 lib/librte_python/rte_python.c           |  387 +++++++++++
 lib/librte_python/rte_python.h           |   71 ++
 lib/librte_python/rte_python_version.map |   12 +
 mk/rte.app.mk                            |    1 +
 test/expect/init.exp                     |   28 +
 test/expect/rx.exp                       |  134 ++++
 test/test/test_cmdline_lib.c             |   10 +-
 23 files changed, 2199 insertions(+), 10 deletions(-)
 create mode 100644 app/test-pmd/pktgen.c
 create mode 100644 doc/guides/howto/scapy.rst
 create mode 100644 lib/librte_python/Makefile
 create mode 100644 lib/librte_python/rte_python.c
 create mode 100644 lib/librte_python/rte_python.h
 create mode 100644 lib/librte_python/rte_python_version.map
 create mode 100644 test/expect/init.exp
 create mode 100644 test/expect/rx.exp

-- 
2.13.3

[dpdk-dev] [RFC v1 00/11] scapy/python extension

[Xueming Li]

This patch introduced a set of new elements into DPDK:
1. python/scapy as a standard dpdk library, disabled by default.
   Could be used by any application to integrate powerful python features.
2. tx, rx and expect, blocking mode testpmd CLI command
   tx command send packets with scapy syntax template, support range/enum, be able to generate multiple flows in one command and DPDK speed.
   rx command receive packets with timeout, with several choices on how to dump packets and mbuf headers.
   expect command try to send out packet, receive and compare with the packet sent, support mbuf header and packet header comparation, show detail info or hexdiff if anything different. Expect command is suitable for unit test and regression test by saving batch expect scripts into a file and use 'load' command to invoke them together.
3. pktgen - new testpmd forwarding engine
  Used to support tx, rx and expect blocking command.
  Also support rxonly/loopback/forward/macswap non-blocking idle modes with several packet dump choice.
4. py command - new testpmd CLI
   Run and evaluate python clause in a global context, samples:
    - py a = 123; b=0xaaa; hex(a+b)
    - py eth=Ether();ip=IP();
    - py hexdump(eth/ip/UDP());
    - tx 0 eth/ip
   py shell: sneak into python shell

The purpose of this scapy/python integration is to help programmers to speed up unit test w/o writing complex c code, verify features with batch expect scripts. Testpmd already a widely used platform for people to setup PMD and verify features with rich CLI command, that's why I choose to make new CLI on this familier tool instead of creating a new one.

I made hundreds of expect test cases when developing rte_flow vxlan and GRE features, that makes me pretty confident to touch any code, hope this tool help you as well.

quick guide document:
	https://github.com/steevenlee/dpdk/blob/master_scapy/doc/guides/howto/scapy.rst
github branch:
        https://github.com/steevenlee/dpdk/tree/master_scapy



Xueming Li (11):
  lib/cmdline: support backspace key
  lib/cmdline: init parse result memeory
  lib/cmdline: add echo support in batch loading from file
  app/testpmd: support command echo in CLI batch loading
  test: update batch loading test
  lib/python: add embedded python lib
  app/testpmd: add python command
  app/testpmd: add pktgen forwarding engine
  app/testpmd: add pktgen engine scapy commands
  test/expect: add expect test scripts
  doc/scapy: add scapy how-to guide

 app/test-pmd/Makefile                    |    6 +
 app/test-pmd/cmdline.c                   |   80 ++-
 app/test-pmd/pktgen.c                    | 1092 ++++++++++++++++++++++++++++++
 app/test-pmd/testpmd.c                   |    1 +
 app/test-pmd/testpmd.h                   |    5 +
 config/common_base                       |    6 +
 doc/guides/howto/scapy.rst               |  300 ++++++++
 lib/Makefile                             |    2 +
 lib/librte_cmdline/cmdline_parse.c       |    2 +
 lib/librte_cmdline/cmdline_rdline.c      |    1 +
 lib/librte_cmdline/cmdline_socket.c      |    5 +-
 lib/librte_cmdline/cmdline_socket.h      |    3 +-
 lib/librte_cmdline/cmdline_vt100.c       |    1 +
 lib/librte_cmdline/cmdline_vt100.h       |    1 +
 lib/librte_eal/common/include/rte_log.h  |    1 +
 lib/librte_python/Makefile               |   60 ++
 lib/librte_python/rte_python.c           |  387 +++++++++++
 lib/librte_python/rte_python.h           |   71 ++
 lib/librte_python/rte_python_version.map |   12 +
 mk/rte.app.mk                            |    1 +
 test/expect/init.exp                     |   28 +
 test/expect/rx.exp                       |  134 ++++
 test/test/test_cmdline_lib.c             |   10 +-
 23 files changed, 2199 insertions(+), 10 deletions(-)
 create mode 100644 app/test-pmd/pktgen.c
 create mode 100644 doc/guides/howto/scapy.rst
 create mode 100644 lib/librte_python/Makefile
 create mode 100644 lib/librte_python/rte_python.c
 create mode 100644 lib/librte_python/rte_python.h
 create mode 100644 lib/librte_python/rte_python_version.map
 create mode 100644 test/expect/init.exp
 create mode 100644 test/expect/rx.exp

-- 
2.13.3

Re: [dpdk-dev] [RFC v1 00/11] scapy/python extension

[Xueming(Steven) Li]

Sorry for spamming, something wrong to post patches.
Please refer to https://github.com/steevenlee/dpdk/tree/master_scapy for code detail.

> -----Original Message-----
> From: Xueming Li [mailto:xuemingl@mellanox.com]
> Sent: Tuesday, December 5, 2017 12:55 PM
> To: Wu@dev-r630-06.mtbc.labs.mlnx; Jingjing <jingjing.wu@intel.com>; Harry
> van Haaren <harry.van.haaren@intel.com>
> Cc: Xueming(Steven) Li <xuemingl@mellanox.com>; dev@dpdk.org
> Subject: [RFC v1 00/11] scapy/python extension
> 
> This patch introduced a set of new elements into DPDK:
> 1. python/scapy as a standard dpdk library, disabled by default.
>    Could be used by any application to integrate powerful python features.
> 2. tx, rx and expect, blocking mode testpmd CLI command
>    tx command send packets with scapy syntax template, support range/enum,
> be able to generate multiple flows in one command and DPDK speed.
>    rx command receive packets with timeout, with several choices on how to
> dump packets and mbuf headers.
>    expect command try to send out packet, receive and compare with the
> packet sent, support mbuf header and packet header comparation, show
> detail info or hexdiff if anything different. Expect command is suitable
> for unit test and regression test by saving batch expect scripts into a
> file and use 'load' command to invoke them together.
> 3. pktgen - new testpmd forwarding engine
>   Used to support tx, rx and expect blocking command.
>   Also support rxonly/loopback/forward/macswap non-blocking idle modes
> with several packet dump choice.
> 4. py command - new testpmd CLI
>    Run and evaluate python clause in a global context, samples:
>     - py a = 123; b=0xaaa; hex(a+b)
>     - py eth=Ether();ip=IP();
>     - py hexdump(eth/ip/UDP());
>     - tx 0 eth/ip
>    py shell: sneak into python shell
> 
> The purpose of this scapy/python integration is to help programmers to
> speed up unit test w/o writing complex c code, verify features with batch
> expect scripts. Testpmd already a widely used platform for people to setup
> PMD and verify features with rich CLI command, that's why I choose to make
> new CLI on this familier tool instead of creating a new one.
> 
> I made hundreds of expect test cases when developing rte_flow vxlan and
> GRE features, that makes me pretty confident to touch any code, hope this
> tool help you as well.
> 
> quick guide document:
> 	https://github.com/steevenlee/dpdk/blob/master_scapy/doc/guides/howt
> o/scapy.rst
> github branch:
>         https://github.com/steevenlee/dpdk/tree/master_scapy
> 
> 
> 
> Xueming Li (11):
>   lib/cmdline: support backspace key
>   lib/cmdline: init parse result memeory
>   lib/cmdline: add echo support in batch loading from file
>   app/testpmd: support command echo in CLI batch loading
>   test: update batch loading test
>   lib/python: add embedded python lib
>   app/testpmd: add python command
>   app/testpmd: add pktgen forwarding engine
>   app/testpmd: add pktgen engine scapy commands
>   test/expect: add expect test scripts
>   doc/scapy: add scapy how-to guide
> 
>  app/test-pmd/Makefile                    |    6 +
>  app/test-pmd/cmdline.c                   |   80 ++-
>  app/test-pmd/pktgen.c                    | 1092
> ++++++++++++++++++++++++++++++
>  app/test-pmd/testpmd.c                   |    1 +
>  app/test-pmd/testpmd.h                   |    5 +
>  config/common_base                       |    6 +
>  doc/guides/howto/scapy.rst               |  300 ++++++++
>  lib/Makefile                             |    2 +
>  lib/librte_cmdline/cmdline_parse.c       |    2 +
>  lib/librte_cmdline/cmdline_rdline.c      |    1 +
>  lib/librte_cmdline/cmdline_socket.c      |    5 +-
>  lib/librte_cmdline/cmdline_socket.h      |    3 +-
>  lib/librte_cmdline/cmdline_vt100.c       |    1 +
>  lib/librte_cmdline/cmdline_vt100.h       |    1 +
>  lib/librte_eal/common/include/rte_log.h  |    1 +
>  lib/librte_python/Makefile               |   60 ++
>  lib/librte_python/rte_python.c           |  387 +++++++++++
>  lib/librte_python/rte_python.h           |   71 ++
>  lib/librte_python/rte_python_version.map |   12 +
>  mk/rte.app.mk                            |    1 +
>  test/expect/init.exp                     |   28 +
>  test/expect/rx.exp                       |  134 ++++
>  test/test/test_cmdline_lib.c             |   10 +-
>  23 files changed, 2199 insertions(+), 10 deletions(-)  create mode 100644
> app/test-pmd/pktgen.c  create mode 100644 doc/guides/howto/scapy.rst
> create mode 100644 lib/librte_python/Makefile  create mode 100644
> lib/librte_python/rte_python.c  create mode 100644
> lib/librte_python/rte_python.h  create mode 100644
> lib/librte_python/rte_python_version.map
>  create mode 100644 test/expect/init.exp  create mode 100644
> test/expect/rx.exp
> 
> --
> 2.13.3

[dpdk-dev] [RFC v1 00/11] scapy/python extension

[Xueming Li]

This patch introduced a set of new elements into DPDK:
1. python/scapy as a standard dpdk library, disabled by default.
   Could be used by any application to integrate powerful python features.
2. tx, rx and expect, blocking mode testpmd CLI command
   tx command send packets with scapy syntax template, support range/enum, be
      able to generate multiple flows in one command and DPDK speed.
   rx command receive packets with timeout, with several choices on how to dump
      packets and mbuf headers.
   expect command try to send out packet, receive and compare with the packet
      sent, support mbuf header and packet header comparation, show detail info
      or hexdiff if anything different. Expect command is suitable for unit test
      and regression test by saving batch expect scripts into a file and use
      'load' command to invoke them together.
3. pktgen - new testpmd forwarding engine
   Used to support tx, rx and expect blocking command.
   Also support rxonly/loopback/forward/macswap non-blocking idle modes with 
       several packet dump choice.
4. py command - new testpmd CLI
   Run and evaluate python clause in a global context, samples:
    - py a = 123; b=0xaaa; hex(a+b)
    - py eth=Ether();ip=IP();
    - py hexdump(eth/ip/UDP());
    - tx 0 eth/ip
   py shell: sneak into python shell

The purpose of this scapy/python integration is to help programmers to speed up
unit test w/o writing complex c code, verify features with batch expect scripts.
Testpmd already a widely used platform for people to setup PMD and verify 
features with rich CLI command, that's why I choose to make new CLI on this 
familier tool instead of creating a new one.

I made hundreds of expect test cases when developing rte_flow vxlan and GRE 
features, that makes me pretty confident to touch any code, hope this tool 
help you as well.

quick guide document:
https://github.com/steevenlee/dpdk/blob/master_scapy/doc/guides/howto/scapy.rst
github branch:
https://github.com/steevenlee/dpdk/tree/master_scapy


Xueming Li (11):
  lib/cmdline: support backspace key
  lib/cmdline: init parse result memeory
  lib/cmdline: add echo support in batch loading from file
  app/testpmd: support command echo in CLI batch loading
  test: update batch loading test
  lib/python: add embedded python lib
  app/testpmd: add python command
  app/testpmd: add pktgen forwarding engine
  app/testpmd: add pktgen engine scapy commands
  test/expect: add expect test scripts
  doc/scapy: add scapy how-to guide

 app/test-pmd/Makefile                    |    6 +
 app/test-pmd/cmdline.c                   |   80 ++-
 app/test-pmd/pktgen.c                    | 1092 ++++++++++++++++++++++++++++++
 app/test-pmd/testpmd.c                   |    1 +
 app/test-pmd/testpmd.h                   |    5 +
 config/common_base                       |    6 +
 doc/guides/howto/scapy.rst               |  300 ++++++++
 lib/Makefile                             |    2 +
 lib/librte_cmdline/cmdline_parse.c       |    2 +
 lib/librte_cmdline/cmdline_rdline.c      |    1 +
 lib/librte_cmdline/cmdline_socket.c      |    5 +-
 lib/librte_cmdline/cmdline_socket.h      |    3 +-
 lib/librte_cmdline/cmdline_vt100.c       |    1 +
 lib/librte_cmdline/cmdline_vt100.h       |    1 +
 lib/librte_eal/common/include/rte_log.h  |    1 +
 lib/librte_python/Makefile               |   60 ++
 lib/librte_python/rte_python.c           |  387 +++++++++++
 lib/librte_python/rte_python.h           |   71 ++
 lib/librte_python/rte_python_version.map |   12 +
 mk/rte.app.mk                            |    1 +
 test/expect/init.exp                     |   28 +
 test/expect/rx.exp                       |  134 ++++
 test/test/test_cmdline_lib.c             |   10 +-
 23 files changed, 2199 insertions(+), 10 deletions(-)
 create mode 100644 app/test-pmd/pktgen.c
 create mode 100644 doc/guides/howto/scapy.rst
 create mode 100644 lib/librte_python/Makefile
 create mode 100644 lib/librte_python/rte_python.c
 create mode 100644 lib/librte_python/rte_python.h
 create mode 100644 lib/librte_python/rte_python_version.map
 create mode 100644 test/expect/init.exp
 create mode 100644 test/expect/rx.exp

-- 
2.13.3

[dpdk-dev] [RFC v1 00/11] scappy/pythoon extension

[Xueming Li]

This patch introduced a set of new elements into DPDK:
1. python/scapy as a standard dpdk library, disabled by default.
   Could be used by any application to integrate powerful python features.
2. tx, rx and expect, blocking mode testpmd CLI command
   tx command send packets with scapy syntax template, support range/enum, be
      able to generate multiple flows in one command and DPDK speed.
   rx command receive packets with timeout, with several choices on how to dump
      packets and mbuf headers.
   expect command try to send out packet, receive and compare with the packet
      sent, support mbuf header and packet header comparation, show detail info
      or hexdiff if anything different. Expect command is suitable for unit test
      and regression test by saving batch expect scripts into a file and use
      'load' command to invoke them together.
3. pktgen - new testpmd forwarding engine
   Used to support tx, rx and expect blocking command.
   Also support rxonly/loopback/forward/macswap non-blocking idle modes with 
       several packet dump choice.
4. py command - new testpmd CLI
   Run and evaluate python clause in a global context, samples:
    - py a = 123; b=0xaaa; hex(a+b)
    - py eth=Ether();ip=IP();
    - py hexdump(eth/ip/UDP());
    - tx 0 eth/ip
   py shell: sneak into python shell

The purpose of this scapy/python integration is to help programmers to speed up
unit test w/o writing complex c code, verify features with batch expect scripts.
Testpmd already a widely used platform for people to setup PMD and verify 
features with rich CLI command, that's why I choose to make new CLI on this 
familier tool instead of creating a new one.

I made hundreds of expect test cases when developing rte_flow vxlan and GRE 
features, that makes me pretty confident to touch any code, hope this tool 
help you as well.

quick guide document:
https://github.com/steevenlee/dpdk/blob/master_scapy/doc/guides/howto/scapy.rst
github branch:
https://github.com/steevenlee/dpdk/tree/master_scapy


Xueming Li (11):
  lib/cmdline: support backspace key
  lib/cmdline: init parse result memeory
  lib/cmdline: add echo support in batch loading from file
  app/testpmd: support command echo in CLI batch loading
  test: update batch loading test
  lib/python: add embedded python lib
  app/testpmd: add python command
  app/testpmd: add pktgen forwarding engine
  app/testpmd: add pktgen engine scapy commands
  test/expect: add expect test scripts
  doc/scapy: add scapy how-to guide

 app/test-pmd/Makefile                    |    6 +
 app/test-pmd/cmdline.c                   |   80 ++-
 app/test-pmd/pktgen.c                    | 1092 ++++++++++++++++++++++++++++++
 app/test-pmd/testpmd.c                   |    1 +
 app/test-pmd/testpmd.h                   |    5 +
 config/common_base                       |    6 +
 doc/guides/howto/scapy.rst               |  300 ++++++++
 lib/Makefile                             |    2 +
 lib/librte_cmdline/cmdline_parse.c       |    2 +
 lib/librte_cmdline/cmdline_rdline.c      |    1 +
 lib/librte_cmdline/cmdline_socket.c      |    5 +-
 lib/librte_cmdline/cmdline_socket.h      |    3 +-
 lib/librte_cmdline/cmdline_vt100.c       |    1 +
 lib/librte_cmdline/cmdline_vt100.h       |    1 +
 lib/librte_eal/common/include/rte_log.h  |    1 +
 lib/librte_python/Makefile               |   60 ++
 lib/librte_python/rte_python.c           |  387 +++++++++++
 lib/librte_python/rte_python.h           |   71 ++
 lib/librte_python/rte_python_version.map |   12 +
 mk/rte.app.mk                            |    1 +
 test/expect/init.exp                     |   28 +
 test/expect/rx.exp                       |  134 ++++
 test/test/test_cmdline_lib.c             |   10 +-
 23 files changed, 2199 insertions(+), 10 deletions(-)
 create mode 100644 app/test-pmd/pktgen.c
 create mode 100644 doc/guides/howto/scapy.rst
 create mode 100644 lib/librte_python/Makefile
 create mode 100644 lib/librte_python/rte_python.c
 create mode 100644 lib/librte_python/rte_python.h
 create mode 100644 lib/librte_python/rte_python_version.map
 create mode 100644 test/expect/init.exp
 create mode 100644 test/expect/rx.exp

-- 
2.13.3

[dpdk-dev] [RFC v1 00/11] scapy/python extension

[Xueming Li]

quick guide document:
https://github.com/steevenlee/dpdk/blob/master_scapy/doc/guides/howto/scapy.rst
github branch:
https://github.com/steevenlee/dpdk/tree/master_scapy


Xueming Li (11):
  lib/cmdline: support backspace key
  lib/cmdline: init parse result memeory
  lib/cmdline: add echo support in batch loading from file
  app/testpmd: support command echo in CLI batch loading
  test: update batch loading test
  lib/python: add embedded python lib
  app/testpmd: add python command
  app/testpmd: add pktgen forwarding engine
  app/testpmd: add pktgen engine scapy commands
  test/expect: add expect test scripts
  doc/scapy: add scapy how-to guide

 app/test-pmd/Makefile                    |    6 +
 app/test-pmd/cmdline.c                   |   80 ++-
 app/test-pmd/pktgen.c                    | 1092 ++++++++++++++++++++++++++++++
 app/test-pmd/testpmd.c                   |    1 +
 app/test-pmd/testpmd.h                   |    5 +
 config/common_base                       |    6 +
 doc/guides/howto/scapy.rst               |  300 ++++++++
 lib/Makefile                             |    2 +
 lib/librte_cmdline/cmdline_parse.c       |    2 +
 lib/librte_cmdline/cmdline_rdline.c      |    1 +
 lib/librte_cmdline/cmdline_socket.c      |    5 +-
 lib/librte_cmdline/cmdline_socket.h      |    3 +-
 lib/librte_cmdline/cmdline_vt100.c       |    1 +
 lib/librte_cmdline/cmdline_vt100.h       |    1 +
 lib/librte_eal/common/include/rte_log.h  |    1 +
 lib/librte_python/Makefile               |   60 ++
 lib/librte_python/rte_python.c           |  387 +++++++++++
 lib/librte_python/rte_python.h           |   71 ++
 lib/librte_python/rte_python_version.map |   12 +
 mk/rte.app.mk                            |    1 +
 test/expect/init.exp                     |   28 +
 test/expect/rx.exp                       |  134 ++++
 test/test/test_cmdline_lib.c             |   10 +-
 23 files changed, 2199 insertions(+), 10 deletions(-)
 create mode 100644 app/test-pmd/pktgen.c
 create mode 100644 doc/guides/howto/scapy.rst
 create mode 100644 lib/librte_python/Makefile
 create mode 100644 lib/librte_python/rte_python.c
 create mode 100644 lib/librte_python/rte_python.h
 create mode 100644 lib/librte_python/rte_python_version.map
 create mode 100644 test/expect/init.exp
 create mode 100644 test/expect/rx.exp

-- 
2.13.3

Re: [dpdk-dev] [RFC v1 00/11] scapy/python extension

[Wiles, Keith]

> On Dec 5, 2017, at 12:04 AM, Xueming Li <xuemingl@mellanox.com> wrote:
> 
> quick guide document:
> https://github.com/steevenlee/dpdk/blob/master_scapy/doc/guides/howto/scapy.rst
> github branch:
> https://github.com/steevenlee/dpdk/tree/master_scapy
> 
> 
> Xueming Li (11):
>  lib/cmdline: support backspace key
>  lib/cmdline: init parse result memeory
>  lib/cmdline: add echo support in batch loading from file
>  app/testpmd: support command echo in CLI batch loading
>  test: update batch loading test
>  lib/python: add embedded python lib
>  app/testpmd: add python command
>  app/testpmd: add pktgen forwarding engine
>  app/testpmd: add pktgen engine scapy commands
>  test/expect: add expect test scripts
>  doc/scapy: add scapy how-to guide
> 

Not to be a pain, but using the name pktgen is going to cause a bit of confusion with my Pktgen-DPDK application. Could you change the name to something else like trafficgen or whatever you like? I believe the naming of this feature is going to cause a lot of people to think this ‘pktgen’ is mine and I already have some confusion with the linux kernel traffic generator called ‘pktgen’ which is why I try to make sure everyone uses Pktgen-DPDK for the name of mine.

It would be very helpful if you would change the name from pktgen to something else, thanks.

> app/test-pmd/Makefile                    |    6 +
> app/test-pmd/cmdline.c                   |   80 ++-
> app/test-pmd/pktgen.c                    | 1092 ++++++++++++++++++++++++++++++
> app/test-pmd/testpmd.c                   |    1 +
> app/test-pmd/testpmd.h                   |    5 +
> config/common_base                       |    6 +
> doc/guides/howto/scapy.rst               |  300 ++++++++
> lib/Makefile                             |    2 +
> lib/librte_cmdline/cmdline_parse.c       |    2 +
> lib/librte_cmdline/cmdline_rdline.c      |    1 +
> lib/librte_cmdline/cmdline_socket.c      |    5 +-
> lib/librte_cmdline/cmdline_socket.h      |    3 +-
> lib/librte_cmdline/cmdline_vt100.c       |    1 +
> lib/librte_cmdline/cmdline_vt100.h       |    1 +
> lib/librte_eal/common/include/rte_log.h  |    1 +
> lib/librte_python/Makefile               |   60 ++
> lib/librte_python/rte_python.c           |  387 +++++++++++
> lib/librte_python/rte_python.h           |   71 ++
> lib/librte_python/rte_python_version.map |   12 +
> mk/rte.app.mk                            |    1 +
> test/expect/init.exp                     |   28 +
> test/expect/rx.exp                       |  134 ++++
> test/test/test_cmdline_lib.c             |   10 +-
> 23 files changed, 2199 insertions(+), 10 deletions(-)
> create mode 100644 app/test-pmd/pktgen.c
> create mode 100644 doc/guides/howto/scapy.rst
> create mode 100644 lib/librte_python/Makefile
> create mode 100644 lib/librte_python/rte_python.c
> create mode 100644 lib/librte_python/rte_python.h
> create mode 100644 lib/librte_python/rte_python_version.map
> create mode 100644 test/expect/init.exp
> create mode 100644 test/expect/rx.exp
> 
> -- 
> 2.13.3
> 

Regards,
Keith

Re: [dpdk-dev] [RFC v1 00/11] scapy/python extension

[Eelco Chaudron]

Hi Xueming,

I was wondering if this patch went anywhere, could not find anything in 
the list archive?

I like the idea, as it’s useful for quick unit testing, without 
relying on Trex or trafgen.

Cheers,

Eelco


> quick guide document:
> https://github.com/steevenlee/dpdk/blob/master_scapy/doc/guides/howto/scapy.rst
> github branch:
> https://github.com/steevenlee/dpdk/tree/master_scapy
>
>
> Xueming Li (11):
>   lib/cmdline: support backspace key
>   lib/cmdline: init parse result memeory
>   lib/cmdline: add echo support in batch loading from file
>   app/testpmd: support command echo in CLI batch loading
>   test: update batch loading test
>   lib/python: add embedded python lib
>   app/testpmd: add python command
>   app/testpmd: add pktgen forwarding engine
>   app/testpmd: add pktgen engine scapy commands
>   test/expect: add expect test scripts
>   doc/scapy: add scapy how-to guide
>
>  app/test-pmd/Makefile                    |    6 +
>  app/test-pmd/cmdline.c                   |   80 ++-
>  app/test-pmd/pktgen.c                    | 1092 
> ++++++++++++++++++++++++++++++
>  app/test-pmd/testpmd.c                   |    1 +
>  app/test-pmd/testpmd.h                   |    5 +
>  config/common_base                       |    6 +
>  doc/guides/howto/scapy.rst               |  300 ++++++++
>  lib/Makefile                             |    2 +
>  lib/librte_cmdline/cmdline_parse.c       |    2 +
>  lib/librte_cmdline/cmdline_rdline.c      |    1 +
>  lib/librte_cmdline/cmdline_socket.c      |    5 +-
>  lib/librte_cmdline/cmdline_socket.h      |    3 +-
>  lib/librte_cmdline/cmdline_vt100.c       |    1 +
>  lib/librte_cmdline/cmdline_vt100.h       |    1 +
>  lib/librte_eal/common/include/rte_log.h  |    1 +
>  lib/librte_python/Makefile               |   60 ++
>  lib/librte_python/rte_python.c           |  387 +++++++++++
>  lib/librte_python/rte_python.h           |   71 ++
>  lib/librte_python/rte_python_version.map |   12 +
>  mk/rte.app.mk                            |    1 +
>  test/expect/init.exp                     |   28 +
>  test/expect/rx.exp                       |  134 ++++
>  test/test/test_cmdline_lib.c             |   10 +-
>  23 files changed, 2199 insertions(+), 10 deletions(-)
>  create mode 100644 app/test-pmd/pktgen.c
>  create mode 100644 doc/guides/howto/scapy.rst
>  create mode 100644 lib/librte_python/Makefile
>  create mode 100644 lib/librte_python/rte_python.c
>  create mode 100644 lib/librte_python/rte_python.h
>  create mode 100644 lib/librte_python/rte_python_version.map
>  create mode 100644 test/expect/init.exp
>  create mode 100644 test/expect/rx.exp
>
> -- 
> 2.13.3

Re: [dpdk-dev] [RFC v1 00/11] scapy/python extension

[Xueming(Steven) Li]

Thanks for interesting on this, I'm using it for my daily development and so on some of my customer.

There is a version here, almost my latest code, welcome for pushes.
https://github.com/steevenlee/dpdk/tree/master_scapy


> -----Original Message-----
> From: Eelco Chaudron <echaudro@redhat.com>
> Sent: Thursday, January 10, 2019 9:07 PM
> To: Xueming(Steven) Li <xuemingl@mellanox.com>
> Cc: dev@dpdk.org
> Subject: Re: [dpdk-dev] [RFC v1 00/11] scapy/python extension
> 
> Hi Xueming,
> 
> I was wondering if this patch went anywhere, could not find anything in the list archive?
> 
> I like the idea, as it’s useful for quick unit testing, without relying on Trex or trafgen.
> 
> Cheers,
> 
> Eelco
> 
> 
> > quick guide document:
> > https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgit
> > hub.com%2Fsteevenlee%2Fdpdk%2Fblob%2Fmaster_scapy%2Fdoc%2Fguides%2Fhow
> > to%2Fscapy.rst&amp;data=02%7C01%7Cxuemingl%40mellanox.com%7Cb189685338
> > d84a05990b08d676fc7a82%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C0%7C63
> > 6827224097402827&amp;sdata=WPOdvHM%2BA3x1r2ihqqI2buNmiWCJUEx6Tf1GxSZqW
> > %2F4%3D&amp;reserved=0
> > github branch:
> > https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgit
> > hub.com%2Fsteevenlee%2Fdpdk%2Ftree%2Fmaster_scapy&amp;data=02%7C01%7Cx
> > uemingl%40mellanox.com%7Cb189685338d84a05990b08d676fc7a82%7Ca652971c7d
> > 2e4d9ba6a4d149256f461b%7C0%7C0%7C636827224097402827&amp;sdata=M%2FAqZ3
> > zbehCwLQNKZCm0seUzWs3mIdBrVwv0%2B5NWmk8%3D&amp;reserved=0
> >
> >
> > Xueming Li (11):
> >   lib/cmdline: support backspace key
> >   lib/cmdline: init parse result memeory
> >   lib/cmdline: add echo support in batch loading from file
> >   app/testpmd: support command echo in CLI batch loading
> >   test: update batch loading test
> >   lib/python: add embedded python lib
> >   app/testpmd: add python command
> >   app/testpmd: add pktgen forwarding engine
> >   app/testpmd: add pktgen engine scapy commands
> >   test/expect: add expect test scripts
> >   doc/scapy: add scapy how-to guide
> >
> >  app/test-pmd/Makefile                    |    6 +
> >  app/test-pmd/cmdline.c                   |   80 ++-
> >  app/test-pmd/pktgen.c                    | 1092
> > ++++++++++++++++++++++++++++++
> >  app/test-pmd/testpmd.c                   |    1 +
> >  app/test-pmd/testpmd.h                   |    5 +
> >  config/common_base                       |    6 +
> >  doc/guides/howto/scapy.rst               |  300 ++++++++
> >  lib/Makefile                             |    2 +
> >  lib/librte_cmdline/cmdline_parse.c       |    2 +
> >  lib/librte_cmdline/cmdline_rdline.c      |    1 +
> >  lib/librte_cmdline/cmdline_socket.c      |    5 +-
> >  lib/librte_cmdline/cmdline_socket.h      |    3 +-
> >  lib/librte_cmdline/cmdline_vt100.c       |    1 +
> >  lib/librte_cmdline/cmdline_vt100.h       |    1 +
> >  lib/librte_eal/common/include/rte_log.h  |    1 +
> >  lib/librte_python/Makefile               |   60 ++
> >  lib/librte_python/rte_python.c           |  387 +++++++++++
> >  lib/librte_python/rte_python.h           |   71 ++
> >  lib/librte_python/rte_python_version.map |   12 +
> >  mk/rte.app.mk                            |    1 +
> >  test/expect/init.exp                     |   28 +
> >  test/expect/rx.exp                       |  134 ++++
> >  test/test/test_cmdline_lib.c             |   10 +-
> >  23 files changed, 2199 insertions(+), 10 deletions(-)  create mode
> > 100644 app/test-pmd/pktgen.c  create mode 100644
> > doc/guides/howto/scapy.rst  create mode 100644
> > lib/librte_python/Makefile  create mode 100644
> > lib/librte_python/rte_python.c  create mode 100644
> > lib/librte_python/rte_python.h  create mode 100644
> > lib/librte_python/rte_python_version.map
> >  create mode 100644 test/expect/init.exp  create mode 100644
> > test/expect/rx.exp
> >
> > --
> > 2.13.3

[dpdk-dev] [RFC v1 0/9] scapy/python extension

[Xueming Li]

This patch introduced a set of new elements into DPDK:
1. python/scapy as a standard dpdk library, disabled by default.
   Could be used by any application to integrate powerful python features.
2. tx, rx and expect, blocking mode testpmd CLI command
   tx command send packets with scapy syntax template, support range/enum, be able to generate multiple flows in one command and DPDK speed.
   rx command receive packets with timeout, with several choices on how to dump packets and mbuf headers.
   expect command try to send out packet, receive and compare with the packet sent, support mbuf header and packet header comparation, show detail info or hexdiff if anything different. Expect command is suitable for unit test and regression test by saving batch expect scripts into a file and use 'load' command to invoke them together.
3. pktgen - new testpmd forwarding engine
  Used to support tx, rx and expect blocking command.
  Also support rxonly/loopback/forward/macswap non-blocking idle modes with several packet dump choice.
4. py command - new testpmd CLI
   Run and evaluate python clause in a global context, samples:
    - py a = 123; b=0xaaa; hex(a+b)
    - py eth=Ether();ip=IP();
    - py hexdump(eth/ip/UDP());
    - tx 0 eth/ip
   py shell: sneak into python shell

The purpose of this scapy/python integration is to help programmers to speed up unit test w/o writing complex c code, verify features with batch expect scripts. Testpmd already a widely used platform for people to setup PMD and verify features with rich CLI command, that's why I choose to make new CLI on this familier tool instead of creating a new one.

I made hundreds of expect test cases when developing rte_flow vxlan and GRE features, that makes me pretty confident to touch any code, hope this tool help you as well.

quick guide document:
	https://github.com/steevenlee/dpdk/blob/master_scapy/doc/guides/howto/scapy.rst
github branch:
        https://github.com/steevenlee/dpdk/tree/master_scapy

v2:
* Introduced rx, tx, expect command
* Introduced pktgen engine idle mode
* Introduced python to dpdk as a std library

Xueming Li (9):
  lib/cmdline: add echo support in batch loading from file
  app/testpmd: support command echo in CLI batch loading
  test: update batch loading test
  lib/python: add embedded python lib
  app/testpmd: add python command
  app/testpmd: add pktgen forwarding engine
  app/testpmd: add pktgen engine scapy commands
  test/expect: add expect test scripts
  doc/scapy: add scapy how-to guide

 app/test-pmd/Makefile                    |    6 +
 app/test-pmd/cmdline.c                   |   80 ++-
 app/test-pmd/pktgen.c                    | 1092 ++++++++++++++++++++++++++++++
 app/test-pmd/testpmd.c                   |    1 +
 app/test-pmd/testpmd.h                   |    5 +
 config/common_base                       |    6 +
 doc/guides/howto/scapy.rst               |  300 ++++++++
 lib/Makefile                             |    2 +
 lib/librte_cmdline/cmdline_socket.c      |    5 +-
 lib/librte_cmdline/cmdline_socket.h      |    3 +-
 lib/librte_eal/common/include/rte_log.h  |    1 +
 lib/librte_python/Makefile               |   60 ++
 lib/librte_python/rte_python.c           |  387 +++++++++++
 lib/librte_python/rte_python.h           |   71 ++
 lib/librte_python/rte_python_version.map |   12 +
 mk/rte.app.mk                            |    1 +
 test/expect/init.exp                     |   28 +
 test/expect/rx.exp                       |  134 ++++
 test/test/test_cmdline_lib.c             |   10 +-
 19 files changed, 2194 insertions(+), 10 deletions(-)
 create mode 100644 app/test-pmd/pktgen.c
 create mode 100644 doc/guides/howto/scapy.rst
 create mode 100644 lib/librte_python/Makefile
 create mode 100644 lib/librte_python/rte_python.c
 create mode 100644 lib/librte_python/rte_python.h
 create mode 100644 lib/librte_python/rte_python_version.map
 create mode 100644 test/expect/init.exp
 create mode 100644 test/expect/rx.exp

-- 
2.13.3

[dpdk-dev] [RFC v1 1/9] lib/cmdline: add echo support in batch loading from file

[Xueming Li]

Add echo option to echo commandline to screen when running loaded
scripts from file.

Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 lib/librte_cmdline/cmdline_socket.c | 5 +++--
 lib/librte_cmdline/cmdline_socket.h | 3 ++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/librte_cmdline/cmdline_socket.c b/lib/librte_cmdline/cmdline_socket.c
index 3fc243b70..e57ddeffb 100644
--- a/lib/librte_cmdline/cmdline_socket.c
+++ b/lib/librte_cmdline/cmdline_socket.c
@@ -73,7 +73,8 @@
 #include "cmdline.h"
 
 struct cmdline *
-cmdline_file_new(cmdline_parse_ctx_t *ctx, const char *prompt, const char *path)
+cmdline_file_new(cmdline_parse_ctx_t *ctx, const char *prompt, const char *path,
+		 int echo)
 {
 	int fd;
 
@@ -86,7 +87,7 @@ cmdline_file_new(cmdline_parse_ctx_t *ctx, const char *prompt, const char *path)
 		dprintf("open() failed\n");
 		return NULL;
 	}
-	return cmdline_new(ctx, prompt, fd, -1);
+	return cmdline_new(ctx, prompt, fd, echo ? 1 : -1);
 }
 
 struct cmdline *
diff --git a/lib/librte_cmdline/cmdline_socket.h b/lib/librte_cmdline/cmdline_socket.h
index aa6068e7e..208134b12 100644
--- a/lib/librte_cmdline/cmdline_socket.h
+++ b/lib/librte_cmdline/cmdline_socket.h
@@ -68,7 +68,8 @@
 extern "C" {
 #endif
 
-struct cmdline *cmdline_file_new(cmdline_parse_ctx_t *ctx, const char *prompt, const char *path);
+struct cmdline *cmdline_file_new(cmdline_parse_ctx_t *ctx, const char *prompt,
+				 const char *path, int echo);
 struct cmdline *cmdline_stdin_new(cmdline_parse_ctx_t *ctx, const char *prompt);
 void cmdline_stdin_exit(struct cmdline *cl);
 
-- 
2.13.3

[dpdk-dev] [RFC v1 2/9] app/testpmd: support command echo in CLI batch loading

[Xueming Li]

Use first bit of verbose_level to enable CLI echo of batch loading.

Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 app/test-pmd/cmdline.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index f71d96301..4acce9f07 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -15778,7 +15778,8 @@ cmdline_read_from_file(const char *filename)
 {
 	struct cmdline *cl;
 
-	cl = cmdline_file_new(main_ctx, "testpmd> ", filename);
+	cl = cmdline_file_new(main_ctx, "testpmd> ", filename,
+			      verbose_level & 0x8000);
 	if (cl == NULL) {
 		printf("Failed to create file based cmdline context: %s\n",
 		       filename);
-- 
2.13.3

[dpdk-dev] [RFC v1 3/9] test: update batch loading test

[Xueming Li]

Support echo back in batch loading.

Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 test/test/test_cmdline_lib.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/test/test_cmdline_lib.c b/test/test/test_cmdline_lib.c
index 65b823a72..c700671d1 100644
--- a/test/test/test_cmdline_lib.c
+++ b/test/test/test_cmdline_lib.c
@@ -165,17 +165,17 @@ test_cmdline_socket_fns(void)
 		goto error;
 	if (cmdline_stdin_new(&ctx, NULL) != NULL)
 		goto error;
-	if (cmdline_file_new(NULL, "prompt", "/dev/null") != NULL)
+	if (cmdline_file_new(NULL, "prompt", "/dev/null", 0) != NULL)
 		goto error;
-	if (cmdline_file_new(&ctx, NULL, "/dev/null") != NULL)
+	if (cmdline_file_new(&ctx, NULL, "/dev/null", 0) != NULL)
 		goto error;
-	if (cmdline_file_new(&ctx, "prompt", NULL) != NULL)
+	if (cmdline_file_new(&ctx, "prompt", NULL, 0) != NULL)
 		goto error;
-	if (cmdline_file_new(&ctx, "prompt", "-/invalid/~/path") != NULL) {
+	if (cmdline_file_new(&ctx, "prompt", "-/invalid/~/path", 0) != NULL) {
 		printf("Error: succeeded in opening invalid file for reading!");
 		return -1;
 	}
-	if (cmdline_file_new(&ctx, "prompt", "/dev/null") == NULL) {
+	if (cmdline_file_new(&ctx, "prompt", "/dev/null", 0) == NULL) {
 		printf("Error: failed to open /dev/null for reading!");
 		return -1;
 	}
-- 
2.13.3

[dpdk-dev] [RFC v1 4/9] lib/python: add embedded python lib

[Xueming Li]

Add embedded python lib to support following:
1. Generate packet using scapy syntax
2. Dump packet content with scapy
3. Run python cmd
4. Interactive shell

Conflicts:
	lib/Makefile
	lib/librte_eal/common/include/rte_log.h
---
 config/common_base                       |   6 +
 lib/Makefile                             |   2 +
 lib/librte_eal/common/include/rte_log.h  |   1 +
 lib/librte_python/Makefile               |  60 +++++
 lib/librte_python/rte_python.c           | 387 +++++++++++++++++++++++++++++++
 lib/librte_python/rte_python.h           |  71 ++++++
 lib/librte_python/rte_python_version.map |  12 +
 mk/rte.app.mk                            |   1 +
 8 files changed, 540 insertions(+)
 create mode 100644 lib/librte_python/Makefile
 create mode 100644 lib/librte_python/rte_python.c
 create mode 100644 lib/librte_python/rte_python.h
 create mode 100644 lib/librte_python/rte_python_version.map

diff --git a/config/common_base b/config/common_base
index b8ee8f91c..e53519392 100644
--- a/config/common_base
+++ b/config/common_base
@@ -752,6 +752,12 @@ CONFIG_RTE_LIBRTE_DISTRIBUTOR=y
 CONFIG_RTE_LIBRTE_REORDER=y
 
 #
+# Compile the python library
+#
+CONFIG_RTE_LIBRTE_PYTHON=n
+CONFIG_RTE_LIBRTE_PYTHON_VERSION=python2.7
+
+#
 # Compile librte_port
 #
 CONFIG_RTE_LIBRTE_PORT=y
diff --git a/lib/Makefile b/lib/Makefile
index dc4e8df70..2ed28932f 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -115,6 +115,8 @@ DEPDIRS-librte_pipeline := librte_eal librte_mempool librte_mbuf
 DEPDIRS-librte_pipeline += librte_table librte_port
 DIRS-$(CONFIG_RTE_LIBRTE_REORDER) += librte_reorder
 DEPDIRS-librte_reorder := librte_eal librte_mempool librte_mbuf
+DIRS-$(CONFIG_RTE_LIBRTE_PYTHON) += librte_python
+DEPDIRS-librte_python := librte_eal librte_mempool librte_mbuf librte_ether
 DIRS-$(CONFIG_RTE_LIBRTE_PDUMP) += librte_pdump
 DEPDIRS-librte_pdump := librte_eal librte_mempool librte_mbuf librte_ether
 DIRS-$(CONFIG_RTE_LIBRTE_GSO) += librte_gso
diff --git a/lib/librte_eal/common/include/rte_log.h b/lib/librte_eal/common/include/rte_log.h
index 6c2d35667..27b22bedd 100644
--- a/lib/librte_eal/common/include/rte_log.h
+++ b/lib/librte_eal/common/include/rte_log.h
@@ -88,6 +88,7 @@ extern struct rte_logs rte_logs;
 #define RTE_LOGTYPE_EFD       18 /**< Log related to EFD. */
 #define RTE_LOGTYPE_EVENTDEV  19 /**< Log related to eventdev. */
 #define RTE_LOGTYPE_GSO       20 /**< Log related to GSO. */
+#define RTE_LOGTYPE_PYTHON    21 /**< Log related to python. */
 
 /* these log types can be used in an application */
 #define RTE_LOGTYPE_USER1     24 /**< User-defined log type 1. */
diff --git a/lib/librte_python/Makefile b/lib/librte_python/Makefile
new file mode 100644
index 000000000..cbde17193
--- /dev/null
+++ b/lib/librte_python/Makefile
@@ -0,0 +1,60 @@
+#   BSD LICENSE
+#
+#   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Intel Corporation nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_python.a
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR)
+LDLIBS += -l$(CONFIG_RTELIB_PYTHON_VERSION)
+CFLAGS += -I/usr/include/$(CONFIG_RTE_LIBRTE_PYTHON_VERSION)
+
+
+EXPORT_MAP := rte_python_version.map
+
+LIBABIVER := 1
+
+# all source are stored in SRCS-y
+SRCS-$(CONFIG_RTE_LIBRTE_PYTHON) := rte_python.c
+
+# install this header file
+SYMLINK-$(CONFIG_RTE_LIBRTE_PYTHON)-include := rte_python.h
+
+# this lib depends upon:
+DEPDIRS-$(CONFIG_RTE_LIBRTE_PYTHON) += lib/librte_mbuf
+DEPDIRS-$(CONFIG_RTE_LIBRTE_PYTHON) += lib/librte_mempool
+DEPDIRS-$(CONFIG_RTE_LIBRTE_PYTHON) += lib/librte_eal
+
+
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/lib/librte_python/rte_python.c b/lib/librte_python/rte_python.c
new file mode 100644
index 000000000..6a88cf5f4
--- /dev/null
+++ b/lib/librte_python/rte_python.c
@@ -0,0 +1,387 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <inttypes.h>
+#include <string.h>
+
+#include <rte_log.h>
+#include <rte_mbuf.h>
+#include <rte_errno.h>
+#include <rte_malloc.h>
+
+#include "rte_python.h"
+#include <python2.7/Python.h>
+#include <python2.7/object.h>
+#include <python2.7/dictobject.h>
+#include <python2.7/stringobject.h>
+
+
+typedef PyObject *PyPtr;
+
+
+#define PKT_MIN_LEN	60
+
+#define DBG(o) py_priv_debug(# o, o);
+
+#define CHK(o, expr)									\
+	do {										\
+		o = (expr);								\
+		if (!o) {								\
+			RTE_LOG(WARNING, PYTHON, "%s = %s: null\n", # o, # expr);	\
+			goto err;							\
+		}									\
+	} while (0);
+
+#define CHKP(o, expr) do {CHK(o, expr); DBG(o);} while (0);
+
+#define CHKZ(o, expr) 									\
+	do {										\
+		o = (expr);								\
+		if (o) {								\
+			RTE_LOG(WARNING, PYTHON, "%s = %s: not null\n", # o, # expr);	\
+			goto err;							\
+		}									\
+	} while (0);
+
+struct {
+	PyPtr mod;		/* Default module */
+	PyPtr dict_mod;	/* Module dict */
+	PyPtr dict_local;	/* Local variable */
+	PyPtr cls_ether;	/* Ether class */
+	PyPtr cls_packet;	/* Packet class */
+	PyPtr func_hexdump;	/* hexdump function */
+	PyPtr func_hexdiff;	/* hexdiff function */
+} g;
+
+static rte_spinlock_t lock;
+
+int rte_python_debug = 0;
+
+
+static inline void
+py_priv_debug(const char *name, PyObject *o)
+{
+	if (o && rte_python_debug)
+		RTE_LOG(DEBUG, PYTHON, "%s(%ld): %s\n", name,
+				o->ob_refcnt, o->ob_type->tp_name);
+}
+
+static inline void
+py_priv_check_err(void)
+{
+	if (PyErr_Occurred()) {
+		PyErr_Print();
+		PyErr_Clear();
+	}
+}
+
+static inline void
+py_priv_free(void *ptrs, int size)
+{
+	PyObject *ptr;
+
+	size = size / sizeof(ptr);
+	while(size) {
+		ptr = ((PyObject **)ptrs)[--size];
+		if (ptr)
+			Py_DECREF(ptr);
+	}
+}
+
+void
+rte_python_close(void)
+{
+	rte_spinlock_lock(&lock);
+	Py_Finalize();
+	py_priv_free(&g, sizeof(g));
+	memset(&g, 0, sizeof(g));
+	rte_spinlock_unlock(&lock);
+}
+
+int
+rte_python_init(void)
+{
+	int r = 0;
+
+	if (g.mod)
+		return 0;
+	rte_spinlock_lock(&lock);
+	Py_Initialize();
+	if (!Py_IsInitialized()) {
+		RTE_LOG(ERR, PYTHON, "Failed to init python\n");
+		goto err;
+	}
+	CHKZ(r, PyRun_SimpleString("import sys;from scapy.all import *"));
+	CHKP(g.mod, PyImport_AddModule("__main__"));
+	CHKP(g.cls_ether, PyObject_GetAttrString(g.mod, "Ether"));
+	CHKP(g.cls_packet, PyObject_GetAttrString(g.mod, "Packet"));
+	CHKP(g.func_hexdump, PyObject_GetAttrString(g.mod, "hexdump"));
+	CHKP(g.func_hexdiff, PyObject_GetAttrString(g.mod, "hexdiff"));
+	CHKP(g.dict_mod, PyModule_GetDict(g.mod));
+	CHKP(g.dict_local, PyDict_New());
+	rte_spinlock_unlock(&lock);
+	return 0;
+err:
+	py_priv_check_err();
+	rte_spinlock_unlock(&lock);
+	rte_python_close();
+	return -1;
+}
+
+int
+rte_python_shell(void)
+{
+	int r;
+
+	rte_spinlock_lock(&lock);
+	r = PyRun_InteractiveLoop(stdin, "<stdin>");
+	rte_spinlock_unlock(&lock);
+	return r;
+}
+
+int
+rte_python_run(char *cmd)
+{
+	struct {
+		PyPtr r;
+	} v = { 0 };
+	int r;
+
+	if (!cmd)
+		return 0;
+	rte_spinlock_lock(&lock);
+	CHKP(v.r, PyRun_String(cmd, Py_single_input, g.dict_mod, g.dict_local));
+	goto end;
+err:
+	r = -1;
+end:
+	py_priv_check_err();
+	py_priv_free(&v, sizeof(v));
+	rte_spinlock_unlock(&lock);
+	return r;
+}
+
+static int
+py_priv_obj_set_mbuf(PyObject *o, struct rte_mbuf *mbuf)
+{
+	char *data;
+	Py_ssize_t len;
+	int r = 0;
+	struct {
+		PyPtr str;
+	} v = { 0 };
+
+	CHKP(v.str, PyObject_Str(o));
+	CHKZ(r, PyString_AsStringAndSize(v.str, &data, &len));
+	/* TODO check max len */
+	rte_memcpy(rte_pktmbuf_mtod_offset(mbuf, void *, 0), data, len);
+	if (len < PKT_MIN_LEN) {
+		memset(rte_pktmbuf_mtod_offset(mbuf, void *, len),
+			0, PKT_MIN_LEN - len + 4);
+		len = PKT_MIN_LEN;
+	}
+	mbuf->pkt_len = len;
+	mbuf->data_len = len;
+	goto end;
+err:
+	r = -1;
+end:
+	py_priv_check_err();
+	py_priv_free(&v, sizeof(v));
+	return r;
+}
+
+struct rte_mbuf *
+rte_python_scapy_to_mbuf(struct rte_mempool *pool, char *pattern)
+{
+	struct {
+		PyPtr pkt;
+	} v = { 0 };
+	struct rte_mbuf *mbuf = NULL;
+	int r;
+
+	RTE_ASSERT(pool && pattern);
+	if (rte_python_init())
+		return NULL;
+	rte_spinlock_lock(&lock);
+	if(!(mbuf = rte_mbuf_raw_alloc(pool))) {
+		RTE_LOG(ERR, PYTHON, "out of memory?");
+		goto err;
+	}
+	CHKP(v.pkt, PyRun_String(pattern, Py_eval_input, g.dict_mod, g.dict_local));
+	CHKZ(r, py_priv_obj_set_mbuf(v.pkt, mbuf));
+	goto end;
+err:
+	if (mbuf)
+		rte_pktmbuf_free(mbuf);
+end:
+	py_priv_check_err();
+	py_priv_free(&v, sizeof(v));
+	rte_spinlock_unlock(&lock);
+	return mbuf;
+}
+
+struct rte_mbuf **
+rte_python_scapy_to_mbufs(struct rte_mempool *pool, char *pattern, uint16_t *count)
+{
+	struct {
+		PyPtr pkts, list;
+	} v = { 0 };
+	struct rte_mbuf **mbufs = NULL;
+	Py_ssize_t size = 0;
+	Py_ssize_t i = 0;
+	int r = 0;
+	int is_list = 0;
+
+	RTE_ASSERT(pool && pattern);
+	if (rte_python_init())
+		return NULL;
+	rte_spinlock_lock(&lock);
+	/* parse python/scapy pattern */
+	CHKP(v.pkts, PyRun_String(pattern, Py_eval_input, g.dict_mod, g.dict_local));
+	/* get size */
+	if (PyIter_Check(v.pkts) || PyList_Check(v.pkts) || PyTuple_Check(v.pkts))
+		is_list = 1;
+	else if (PyObject_IsInstance(v.pkts, g.cls_packet))
+		is_list = 1;
+	if (is_list) {
+		CHKP(v.list, PySequence_Fast(v.pkts, NULL));
+		size = PySequence_Fast_GET_SIZE(v.list);
+	} else
+		size = 1;
+	/* alloc mbufs */
+	if ((mbufs = rte_malloc_socket(NULL, sizeof(void *) * size,
+			0, pool->socket_id)) == NULL ||
+		rte_pktmbuf_alloc_bulk(pool, mbufs, size)) {
+		RTE_LOG(ERR, PYTHON, "out of memory?");
+		goto err;
+	}
+	/* copy to mbufs */
+	if (is_list) {
+		for (i = 0; i < size; ++i)
+			CHKZ(r, py_priv_obj_set_mbuf(
+				PySequence_Fast_GET_ITEM(v.list, i), mbufs[i]));
+		*count = size;
+	} else {
+		CHKZ(r, py_priv_obj_set_mbuf(v.pkts, mbufs[0]));
+		*count = 1;
+	}
+	goto end;
+err:
+	r = -1;
+end:
+	py_priv_check_err();
+	py_priv_free(&v, sizeof(v));
+	if (r && mbufs) {
+		while (size--)
+			if (mbufs[size])
+				rte_pktmbuf_free(mbufs[size]);
+		rte_free(mbufs);
+		mbufs = NULL;
+	}
+	rte_spinlock_unlock(&lock);
+	return mbufs;
+}
+
+int
+rte_python_scapy_dump(struct rte_mbuf *mbuf, int verbose)
+{
+	struct {
+		PyPtr str, pkt, non, args, summary, args1;
+	} v = { 0 };
+	char *data;
+	int r = 0;
+	char hide_defaults[] = "hide_defaults";
+	char show[] = "show";
+	char summary[] = "summary";
+
+	if (!mbuf || !verbose)
+		return 0;
+	if (rte_python_init())
+		return -1;
+	rte_spinlock_lock(&lock);
+	data = rte_pktmbuf_mtod_offset(mbuf, void *, 0);
+	CHKP(v.str, PyString_FromStringAndSize(data, mbuf->data_len));
+	CHKP(v.args, PyTuple_Pack(1, v.str));
+	CHKP(v.pkt, PyObject_CallObject(g.cls_ether, v.args));
+	CHKP(v.non, PyObject_CallMethod(v.pkt, hide_defaults, NULL));
+	if ((verbose & 0xf) == 1) { /* pkt.summary() */
+		CHKP(v.summary, PyObject_CallMethod(v.pkt, summary, NULL));
+		puts(PyString_AsString(v.summary));
+	} else if ((verbose & 0xf) == 2) { /* repr(pkt) */
+		CHKZ(r, PyObject_Print(v.pkt, stdout, 0));
+		puts("");
+	} else if ((verbose & 0xf) >= 3) { /* pkt.show() */
+		CHKP(v.non, PyObject_CallMethod(v.pkt, show, NULL));
+	}
+	if ((verbose & 0x20)) { /* hexdump(pkt); */
+		CHKP(v.args1, PyTuple_Pack(1, v.pkt));
+		CHKP(v.non, PyObject_Call(g.func_hexdump, v.args1, NULL));
+	}
+	goto end;
+err:
+	printf("Error!\n");
+	r = -1;
+end:
+	py_priv_check_err();
+	py_priv_free(&v, sizeof(v));
+	rte_spinlock_unlock(&lock);
+	return r;
+}
+
+int
+rte_python_scapy_hexdiff(void *src, int slen, void *dst, int dlen)
+{
+	struct {
+		PyPtr sstr, dstr, args, r;
+	} v = { 0 };
+	int r = 0;
+
+	if (!src || !dst)
+		return 0;
+	if (rte_python_init())
+		return -1;
+	rte_spinlock_lock(&lock);
+	CHKP(v.sstr, PyString_FromStringAndSize(src, slen));
+	CHKP(v.dstr, PyString_FromStringAndSize(dst, dlen));
+	CHKP(v.args, PyTuple_Pack(2, v.sstr, v.dstr));
+	CHKP(v.r, PyObject_CallObject(g.func_hexdiff, v.args));
+	goto end;
+err:
+	r = -1;
+end:
+	py_priv_check_err();
+	py_priv_free(&v, sizeof(v));
+	rte_spinlock_unlock(&lock);
+	return r;
+}
diff --git a/lib/librte_python/rte_python.h b/lib/librte_python/rte_python.h
new file mode 100644
index 000000000..f3c2ef1f4
--- /dev/null
+++ b/lib/librte_python/rte_python.h
@@ -0,0 +1,71 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_REORDER_H_
+#define _RTE_REORDER_H_
+
+/**
+ * @file
+ * RTE python
+ *
+ * Python library is a component which is designed to
+ * provide embedded python support, major functions:
+ * 1. use of scapy module to generate or display packet
+ * 2. evaluate mbuf result with expression *
+ */
+
+
+#include <rte_mbuf.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int rte_python_debug;
+
+int rte_python_init(void);
+void rte_python_close(void);
+struct rte_mbuf *rte_python_scapy_to_mbuf(struct rte_mempool *pool,
+					  char *pattern);
+struct rte_mbuf **rte_python_scapy_to_mbufs(struct rte_mempool *pool,
+					    char *pattern, uint16_t *count);
+int rte_python_scapy_dump(struct rte_mbuf *mbuf, int verbose);
+int rte_python_scapy_hexdiff(void *src, int slen, void *dst, int dlen);
+int rte_python_shell(void);
+int rte_python_run(char *cmd);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_REORDER_H_ */
diff --git a/lib/librte_python/rte_python_version.map b/lib/librte_python/rte_python_version.map
new file mode 100644
index 000000000..7c5b05b73
--- /dev/null
+++ b/lib/librte_python/rte_python_version.map
@@ -0,0 +1,12 @@
+DPDK_2.0 {
+	global:
+
+	rte_python_init;
+	rte_python_close;
+	rte_python_scapy_to_mbuf;
+	rte_python_scapy_dump;
+	rte_python_shell;
+	rte_python_eval;
+
+	local: *;
+};
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index 6a6a7452e..b11dd23f5 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -101,6 +101,7 @@ _LDLIBS-$(CONFIG_RTE_DRIVER_MEMPOOL_RING)   += -lrte_mempool_ring
 _LDLIBS-$(CONFIG_RTE_LIBRTE_RING)           += -lrte_ring
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PCI)            += -lrte_pci
 _LDLIBS-$(CONFIG_RTE_LIBRTE_EAL)            += -lrte_eal
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PYTHON)         += -lrte_python -l$(CONFIG_RTE_LIBRTE_PYTHON_VERSION) 
 _LDLIBS-$(CONFIG_RTE_LIBRTE_CMDLINE)        += -lrte_cmdline
 _LDLIBS-$(CONFIG_RTE_LIBRTE_REORDER)        += -lrte_reorder
 _LDLIBS-$(CONFIG_RTE_LIBRTE_SCHED)          += -lrte_sched
-- 
2.13.3

[dpdk-dev] [RFC v1 5/9] app/testpmd: add python command

[Xueming Li]

py shell: invoke python shell, Ctrl+d to quit
py <any>: run python clause

Signed-off-by: Xueming Li <xuemingl@mellanox.com>

Conflicts:
	app/test-pmd/cmdline.c
---
 app/test-pmd/cmdline.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index 4acce9f07..6654a62cb 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -95,6 +95,9 @@
 #ifdef RTE_LIBRTE_I40E_PMD
 #include <rte_pmd_i40e.h>
 #endif
+#ifdef RTE_LIBRTE_PYTHON
+#include <rte_python.h>
+#endif
 #ifdef RTE_LIBRTE_BNXT_PMD
 #include <rte_pmd_bnxt.h>
 #endif
@@ -150,6 +153,8 @@ struct cmd_help_long_result {
 	cmdline_fixed_string_t section;
 };
 
+extern cmdline_parse_ctx_t main_ctx[];
+
 static void cmd_help_long_parsed(void *parsed_result,
                                  struct cmdline *cl,
                                  __attribute__((unused)) void *data)
@@ -15535,6 +15540,58 @@ cmdline_parse_inst_t cmd_load_from_file = {
 	},
 };
 
+#ifdef RTE_LIBRTE_PYTHON
+
+/* "py" command */
+struct cmd_py_run_result {
+	cmdline_fixed_string_t py;
+	cmdline_fixed_string_t code;
+};
+
+cmdline_parse_token_string_t cmd_py_run_py =
+	TOKEN_STRING_INITIALIZER(struct cmd_py_run_result, py, "py");
+cmdline_parse_token_string_t cmd_py_run_code =
+	TOKEN_STRING_INITIALIZER(struct cmd_py_run_result, code, "");
+
+static void
+cmd_py_run_parsed(
+	void *parsed_result,
+	__attribute__((unused)) struct cmdline *cl,
+	__attribute__((unused)) void *data)
+{
+	struct cmd_py_run_result *res = parsed_result;
+
+	if (rte_python_init())
+		return;
+	if (!strcmp("shell", res->code)) {
+		cmdline_stdin_exit(testpmd_cl);
+			if (rte_python_shell())
+				printf("Failed to open python shell!\n");
+			testpmd_cl = cmdline_stdin_new(main_ctx, "testpmd> ");
+
+	} else if (!strcmp("debug", res->code)) {
+		rte_python_debug = 1;
+		rte_log_set_level(RTE_LOGTYPE_PYTHON, RTE_LOG_DEBUG);
+	} else if (!strcmp("nodebug", res->code))
+		rte_python_debug = 0;
+	else
+		rte_python_run(res->code);
+}
+
+cmdline_parse_inst_t cmd_py_run = {
+	.f = cmd_py_run_parsed,
+	.data = NULL,
+	.help_str = "run python code like: hex(123)\n"
+			"\t'py shell' to enter shell (ctrl + d to quit)\n"
+			"\t'py debug|nodebug' to toggle debug output, --log-level=9",
+	.tokens = {
+		(void *)&cmd_py_run_py,
+		(void *)&cmd_py_run_code,
+		NULL,
+	},
+};
+#endif
+
 /* ******************************************************************************** */
 
 /* list of instructions */
@@ -15543,6 +15600,9 @@ cmdline_parse_ctx_t main_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_help_long,
 	(cmdline_parse_inst_t *)&cmd_quit,
 	(cmdline_parse_inst_t *)&cmd_load_from_file,
+#ifdef RTE_LIBRTE_PYTHON
+	(cmdline_parse_inst_t *)&cmd_py_run,
+#endif
 	(cmdline_parse_inst_t *)&cmd_showport,
 	(cmdline_parse_inst_t *)&cmd_showqueue,
 	(cmdline_parse_inst_t *)&cmd_showportall,
-- 
2.13.3

[dpdk-dev] [RFC v1 6/9] app/testpmd: add pktgen forwarding engine

[Xueming Li]

Add new "pktgen" forwarding engine that send and receive packets
acoording to pktgen_task instruction.

Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 app/test-pmd/Makefile  |   1 +
 app/test-pmd/cmdline.c |   3 +
 app/test-pmd/pktgen.c  | 605 +++++++++++++++++++++++++++++++++++++++++++++++++
 app/test-pmd/testpmd.c |   1 +
 app/test-pmd/testpmd.h |   1 +
 5 files changed, 611 insertions(+)
 create mode 100644 app/test-pmd/pktgen.c

diff --git a/app/test-pmd/Makefile b/app/test-pmd/Makefile
index d21308fcd..2610a8d49 100644
--- a/app/test-pmd/Makefile
+++ b/app/test-pmd/Makefile
@@ -55,6 +55,7 @@ SRCS-y += iofwd.c
 SRCS-y += macfwd.c
 SRCS-y += macswap.c
 SRCS-y += flowgen.c
+SRCS-y += pktgen.c
 SRCS-y += rxonly.c
 SRCS-y += txonly.c
 SRCS-y += csumonly.c
diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index 6654a62cb..d0eb00c1f 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -15594,6 +15594,8 @@ cmdline_parse_inst_t cmd_py_run = {
 
 /* ******************************************************************************** */
 
+extern cmdline_parse_inst_t cmd_pktgen_cmd;
+
 /* list of instructions */
 cmdline_parse_ctx_t main_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_help_brief,
@@ -15603,6 +15605,7 @@ cmdline_parse_ctx_t main_ctx[] = {
 #ifdef RTE_LIBRTE_PYTHON
 	(cmdline_parse_inst_t *)&cmd_py_run,
 #endif
+	(cmdline_parse_inst_t *)&cmd_pktgen_cmd,
 	(cmdline_parse_inst_t *)&cmd_showport,
 	(cmdline_parse_inst_t *)&cmd_showqueue,
 	(cmdline_parse_inst_t *)&cmd_showportall,
diff --git a/app/test-pmd/pktgen.c b/app/test-pmd/pktgen.c
new file mode 100644
index 000000000..f4a1b00a9
--- /dev/null
+++ b/app/test-pmd/pktgen.c
@@ -0,0 +1,605 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <fcntl.h>
+
+#include <sys/queue.h>
+#include <sys/stat.h>
+
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_debug.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_per_lcore.h>
+#include <rte_lcore.h>
+#include <rte_atomic.h>
+#include <rte_branch_prediction.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_interrupts.h>
+#include <rte_pci.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_flow.h>
+#include <rte_string_fns.h>
+#include <cmdline.h>
+#include <cmdline_rdline.h>
+#include <cmdline_parse.h>
+#include <cmdline_parse_num.h>
+#include <cmdline_parse_string.h>
+#include <cmdline_parse_ipaddr.h>
+#include <cmdline_parse_etheraddr.h>
+#include <cmdline_socket.h>
+#ifdef RTE_LIBRTE_PYTHON
+#include <rte_python.h>
+#endif
+
+#include "testpmd.h"
+
+#define PKTGEN_ROUND_END 0x1
+#define PKTGEN_TASK_END 0x0
+#define PKTGEN_TASK_START 0x3
+
+struct pktgen_task_stats{
+	uint8_t active;
+	uint16_t round;		/* number txrx */
+	uint64_t count;
+	uint64_t start; 	/* start tsc */
+	uint64_t end;		/* end tsc */
+};
+
+struct pktgen_task {
+	void *data;		/* rx: tx; tx: mbuf[] */
+	uint64_t count;
+	char *field;		/* field to match */
+	uint64_t val;		/* field value */
+	uint16_t round;		/* number txrx */
+	uint16_t cnt_mbufs;	/* number of templates */
+	uint8_t verbose:7;
+	uint8_t txrx:1; 	/* txrx type task */
+	struct pktgen_task_stats stats;
+} __rte_cache_aligned;
+
+static void *pktgen_tx_tasks;
+static void *pktgen_rx_tasks;
+static int pktgen_idle_mode = 2; /* 0-drop 1-loopback 2-forward 3-switch */
+static int pktgen_busy = 0;
+
+static inline struct rte_mbuf *
+pg_task_template_get(struct pktgen_task* task, uint32_t idx)
+{
+	struct rte_mbuf **mbufs = task->data;
+
+	return mbufs ? mbufs[idx % task->cnt_mbufs] : NULL;
+}
+
+static inline struct pktgen_task *
+task_tx(portid_t port, queueid_t queue)
+{
+	RTE_ASSERT(pktgen_tx_tasks);
+	struct pktgen_task (*tasks)[nb_txq] = pktgen_tx_tasks;
+	return &tasks[port][queue];
+}
+
+static inline struct pktgen_task *
+task_rx(portid_t port, queueid_t queue)
+{
+	RTE_ASSERT(pktgen_rx_tasks);
+	struct pktgen_task (*tasks)[nb_rxq] = pktgen_rx_tasks;
+	return &tasks[port][queue];
+}
+
+/********************************************************/
+/* Forwarding thread functions                          */
+/********************************************************/
+
+static inline void
+pg_dump_mbuf_header(struct rte_mbuf *mbuf, struct fwd_stream* fs, int is_rx) {
+	char buf[256];
+
+	printf("%s P:%hu Q:%hu len:%hu ptype:0x%x ol_flags:0x%lx rss:0x%08x fdir:0x%x\n",
+			is_rx ? "RX" : "TX",
+			is_rx ? fs->rx_port : fs->tx_port,
+			is_rx ? fs->rx_queue : fs->tx_queue,
+			mbuf->data_len, mbuf->packet_type,
+			mbuf->ol_flags,
+			mbuf->hash.rss, mbuf->hash.fdir.hi
+			);
+	if (mbuf->packet_type) {
+		rte_get_ptype_name(mbuf->packet_type, buf, sizeof(buf));
+		printf("  ptype: %s\n", buf);
+	}
+	if (mbuf->tx_offload)
+		printf("  header len:%d/%d/%d/%d/%d tso len:%hu\n",
+				mbuf->outer_l2_len, mbuf->outer_l3_len,
+				mbuf->l2_len, mbuf->l3_len, mbuf->l4_len,
+				mbuf->tso_segsz);
+	else if (mbuf->ol_flags){
+		rte_get_rx_ol_flag_list(mbuf->ol_flags, buf, sizeof(buf));
+		printf("  ol_flags: %s\n", buf);
+	}
+}
+
+static inline void
+pg_debug(struct rte_mbuf *mbuf, uint8_t level, struct fwd_stream* fs, int is_rx)
+{
+	/* xxxx xxxx
+	 *   ||	  L- 1: summary 2: repr 3:show
+	 *   |L----- mbuf header
+	 *   L------ hex dump
+	 */
+	if (level & 0x10)
+		pg_dump_mbuf_header(mbuf, fs, is_rx);
+#ifdef RTE_LIBRTE_PYTHON
+	if (level)
+		rte_python_scapy_dump(mbuf, level);
+#else
+	(void) mbuf;
+#endif
+}
+
+static int
+pg_mbuf_field_expect(struct fwd_stream* fs, struct pktgen_task *task,
+		struct rte_mbuf *mbuf)
+{
+	#define OFF(field) offsetof(struct rte_mbuf, field)
+	unsigned int i;
+	uint64_t val;
+	static struct {
+		const char *name;
+		uint8_t offset;
+		uint8_t shift;
+		uint64_t mask;
+	} fields [] = {
+		{"port", 	OFF(port), 		0, UINT16_MAX},
+		{"ptype",	OFF(packet_type), 	0, UINT32_MAX},
+		{"rss", 	OFF(hash.rss),		0, UINT32_MAX},
+		{"fdir", 	OFF(hash.fdir.hi),	0, UINT32_MAX},
+		/* ignore rss bit */
+		{"ol_flags",	OFF(ol_flags), 		0, UINT64_MAX &
+							   ~PKT_RX_RSS_HASH},
+	};
+
+	RTE_ASSERT(task && mbuf);
+	if (!task->field || strlen(task->field) == 0)
+		return 0;
+	if (!strcmp(task->field, "queue")) {
+		if (fs->rx_queue != task->val) {
+			printf("Failed: queue expect: 0x%lu received:0x%hu\n",
+					task->val, fs->rx_queue);
+			return 1;
+		} else
+			return 0;
+	}
+	for (i = 0; i < RTE_DIM(fields); i++) {
+		if (strcmp(task->field, fields[i].name))
+			continue;
+		val = *((uint64_t *)(void *)((char *)(void *)mbuf + fields[i].offset));
+		val &= fields[i].mask;
+		if ((val != (task->val & fields[i].mask))) {
+			printf("Failed: %s mask: 0x%lx expect: 0x%lx received: 0x%lx\n",
+				fields[i].name, fields[i].mask,
+				task->val & fields[i].mask, val);
+			return 1;
+		} else
+			return 0;
+	}
+	printf("Failed: unknown field '%s', valid names: queue,", task->field);
+	for (i = 0; i < RTE_DIM(fields); i++)
+		printf("%s%s", fields[i].name,
+		       i == RTE_DIM(fields) - 1 ? ",non(0)\n" : ",");
+	return 0;
+}
+
+static int
+pg_mbuf_expect(struct fwd_stream* fs, struct pktgen_task *task,
+		struct rte_mbuf *mbuf, uint32_t idx)
+{
+	int r = 0;
+	struct rte_mbuf *exp;
+
+	RTE_ASSERT(task && task->data && mbuf);
+	exp = pg_task_template_get(task->data, idx); /* tx->mbuf */
+	if (!exp) {
+		RTE_LOG(ERR, USER1, "packet tempalte not found, timeout?\n");
+		return -1;
+	}
+	r |= pg_mbuf_field_expect(fs, task, mbuf);
+	if (exp->data_len != mbuf->data_len) {
+		printf("Failed: packet length not same: %hu/%hu",
+				 mbuf->data_len, exp->data_len);
+		r |= 2;
+	} else if (memcmp(
+			rte_pktmbuf_mtod(mbuf, void *),
+			rte_pktmbuf_mtod(exp, void *),
+			mbuf->data_len)) {
+		printf("Failed: packet not same:\n");
+		r |= 4;
+#ifdef RTE_LIBRTE_PYTHON
+		rte_python_scapy_hexdiff(
+			rte_pktmbuf_mtod(exp, void *), exp->data_len,
+			rte_pktmbuf_mtod(mbuf, void *), mbuf->data_len);
+#endif
+	}
+	return r;
+}
+
+static inline void
+pg_mbuf_switch(struct rte_mbuf **pkts, uint16_t nb_to_tx)
+{
+	uint32_t i;
+	struct ether_hdr *eth;
+	struct ether_addr addr;
+
+	for (i = 0; i < nb_to_tx; i++) {
+		eth = rte_pktmbuf_mtod(pkts[i], struct ether_hdr *);
+		ether_addr_copy(&eth->d_addr, &addr);
+		ether_addr_copy(&eth->s_addr, &eth->d_addr);
+		ether_addr_copy(&addr, &eth->s_addr);
+	}
+}
+
+static inline uint16_t
+pg_tx_burst(struct fwd_stream* fs, int mode,
+		struct rte_mbuf **pkts_burst, uint16_t nb_to_tx)
+{
+	uint32_t retry;
+	uint16_t nb_tx, i;
+	portid_t port = mode == 1 ? fs->rx_port : fs->tx_port;
+	queueid_t queue = mode == 1 ? fs->rx_queue : fs->tx_queue;
+
+	if (unlikely(mode == 3))
+		pg_mbuf_switch(pkts_burst, nb_to_tx);
+	nb_tx = rte_eth_tx_burst(port, queue, pkts_burst,
+			nb_to_tx);
+	/* Retry if necessary */
+	if (unlikely(nb_tx < nb_to_tx) && fs->retry_enabled)
+	{
+		retry = 0;
+		while (nb_tx < nb_to_tx && retry++ < burst_tx_retry_num)
+		{
+			rte_delay_us(burst_tx_delay_time);
+			nb_tx += rte_eth_tx_burst(port, queue,
+					&pkts_burst[nb_tx], nb_to_tx - nb_tx);
+		}
+	}
+	/* Drop packets failed to send */
+	if (unlikely(nb_tx < nb_to_tx))
+	{
+		fs->fwd_dropped += (nb_to_tx - nb_tx);
+		i = nb_tx;
+		do {
+			rte_pktmbuf_free(pkts_burst[i]);
+		} while (++i < nb_to_tx);
+	}
+	return nb_tx;
+}
+
+static inline int
+pg_tx_fill(struct rte_mbuf **pkts_burst, uint64_t nb_to_tx,
+		struct pktgen_task *task, struct fwd_stream* fs)
+{
+	uint32_t i;
+	struct rte_mbuf *exp;
+
+	exp = pg_task_template_get(task, task->stats.count);
+	RTE_ASSERT(exp && exp->pool);
+	RTE_ASSERT(task->cnt_mbufs > 0);
+	if (rte_pktmbuf_alloc_bulk(exp->pool,
+			pkts_burst, nb_to_tx))
+		return -1;
+	for (i = 0; i < nb_to_tx; i++) {
+		exp = pg_task_template_get(task, task->stats.count + i);
+		rte_memcpy(rte_pktmbuf_mtod(pkts_burst[i], void *),
+				rte_pktmbuf_mtod(exp, void *),
+				exp->data_len);
+		pkts_burst[i]->pkt_len = exp->pkt_len;
+		pkts_burst[i]->data_len = exp->data_len;
+		pkts_burst[i]->ol_flags = exp->ol_flags;
+		pg_debug(pkts_burst[i], task->verbose, fs, 0);
+	}
+	return 0;
+}
+
+static inline int
+pg_start(struct pktgen_task *task, uint64_t start_tsc)
+{
+	/* even round end, has to check tx stats */
+	if (unlikely(task->stats.active == PKTGEN_TASK_END))
+		return -1;
+	if (!task->stats.start)
+		task->stats.start = start_tsc;
+	return 0;
+}
+
+static inline int
+pg_round_end(struct pktgen_task *task)
+{
+	/* if not txrx task, keep busy */
+	if (unlikely(task->txrx))
+		task->stats.active = PKTGEN_ROUND_END;
+	task->stats.round += 1;
+	return task->round == task->stats.round;
+}
+
+static inline void
+pg_end(struct pktgen_task *task)
+{
+	task->stats.active = PKTGEN_TASK_END;
+	task->stats.end = rte_rdtsc();
+}
+
+/* return -1 if nothing to do */
+static inline int
+pg_tx(struct fwd_stream* fs, struct pktgen_task *task,
+		uint64_t start_tsc)
+{
+	struct rte_mbuf *pkts_burst[nb_pkt_per_burst];
+	uint64_t nb_to_tx = 0;
+	uint64_t nb_tx;
+
+	if (pg_start(task, start_tsc))
+		return -1;
+	if (unlikely(task->stats.active != PKTGEN_TASK_START))
+		return -1;
+	if (task->count) {
+		nb_to_tx = task->count - task->stats.count;
+		if (nb_to_tx > nb_pkt_per_burst)
+			nb_to_tx = nb_pkt_per_burst;
+	} else
+			nb_to_tx = nb_pkt_per_burst;
+	if (likely((nb_to_tx && task->data))) {
+		if (unlikely((pg_tx_fill(pkts_burst, nb_to_tx, task, fs))))
+			return -1;
+		nb_tx = pg_tx_burst(fs, 2, pkts_burst, nb_to_tx);
+		fs->tx_packets += nb_tx;
+		task->stats.count += nb_tx;
+	}
+	if (task->stats.count == task->count) {
+		if (pg_round_end(task)) /* end of taks? */
+			pg_end(task);
+	}
+	return 0;
+}
+
+/* return -1 if nothing to do */
+static inline int
+pg_rx(struct fwd_stream* fs, struct pktgen_task *task, uint64_t start_tsc)
+{
+	struct rte_mbuf *pkts_burst[nb_pkt_per_burst];
+	uint16_t nb_to_rx;
+	uint16_t nb_rx;
+	uint32_t i;
+	uint8_t verbose;
+	struct pktgen_task *tx_task;
+	int r;
+
+	if (pg_start(task, start_tsc))
+		return -1;
+	if (task->count) {
+		nb_to_rx = task->count - task->stats.count;
+		if (nb_to_rx > nb_pkt_per_burst)
+			nb_to_rx = nb_pkt_per_burst;
+	} else /* endless rx */
+		nb_to_rx = nb_pkt_per_burst;
+	if (nb_to_rx && task->stats.active == PKTGEN_TASK_START) {
+		nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue,
+				pkts_burst, nb_pkt_per_burst);
+#ifdef RTE_TEST_PMD_RECORD_BURST_STATS
+		fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
+#endif
+		if (unlikely(nb_rx == 0))
+			return 0;
+		for (i = 0; i < nb_rx; i++) {
+			verbose = task->verbose;
+			if (task->data) {
+				r = pg_mbuf_expect(fs, task, pkts_burst[i], task->stats.count + i);
+				if (r < 0) /* task timeout */
+					return r;
+				else if (r) /* compare failed, simple dump */
+					verbose |= 0x10;
+			}
+			pg_debug(pkts_burst[i], verbose, fs, 1);
+			rte_pktmbuf_free(pkts_burst[i]);
+		}
+		fs->rx_packets += nb_rx;
+		task->stats.count += nb_rx;
+	}
+	if (task->count && task->stats.count >= task->count) {
+		tx_task = task->data;
+		if (task->stats.active == PKTGEN_TASK_START && pg_round_end(task))
+			pg_end(task);
+		else if (tx_task && tx_task->stats.active==PKTGEN_ROUND_END) {
+			/* has tx task, next round */
+			tx_task->stats.active =	PKTGEN_TASK_START;
+			tx_task->stats.count = 0;
+			task->stats.active = PKTGEN_TASK_START;
+			task->stats.count = 0;
+		}
+	}
+	return 0;
+}
+
+static void
+pg_idle_set(int mode)
+{
+	const char *names[] = {
+			"drop(0)",
+			"loopback(1)",
+			"io_forward(2)",
+			"mac_switch(3)"};
+	if (pktgen_idle_mode != mode)
+		printf("PktGen idle mode changed from %s to %s\n",
+		       names[pktgen_idle_mode], names[mode]);
+	pktgen_idle_mode = mode;
+}
+
+static inline int
+pg_rx_idle(struct fwd_stream* fs)
+{
+	struct rte_mbuf *pkts_burst[nb_pkt_per_burst];
+	uint16_t nb_rx;
+	uint32_t i;
+
+	nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue,
+				pkts_burst, nb_pkt_per_burst);
+#ifdef RTE_TEST_PMD_RECORD_BURST_STATS
+	fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
+#endif
+	if (unlikely(nb_rx == 0))
+		return 0;
+	fs->rx_packets += nb_rx;
+	if (verbose_level & 0xff)
+		for (i = 0; i < nb_rx; i++)
+			pg_debug(pkts_burst[i], verbose_level & 0xff, fs, 1);
+	if (pktgen_idle_mode) /* no drop */
+		pg_tx_burst(fs, pktgen_idle_mode, pkts_burst, nb_rx);
+	else /* drop */
+		for (i = 0; i < nb_rx; i++)
+			rte_pktmbuf_free(pkts_burst[i]);
+	return 0;
+}
+
+/*
+ * TX and RX pacets according to traffic generator command.
+ */
+static void
+pg_fwd(struct fwd_stream *fs)
+{
+	uint64_t start;
+#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
+	uint64_t end_tsc;
+	uint64_t core_cycles;
+#endif
+
+	start = rte_rdtsc();
+	if (likely(pktgen_busy)) {
+		pg_tx(fs, task_tx(fs->tx_port, fs->tx_queue), start);
+		pg_rx(fs, task_rx(fs->rx_port, fs->rx_queue), start);
+	} else
+		pg_rx_idle(fs);
+#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
+	end_tsc = rte_rdtsc();
+	core_cycles = (end_tsc - start_tsc);
+	fs->core_cycles += fs->core_cycles + core_cycles;
+#endif
+}
+
+static void
+pktgen_begin(portid_t pi __rte_unused)
+{
+	if (!pktgen_tx_tasks)
+		pktgen_tx_tasks = rte_malloc(NULL,
+			sizeof(struct pktgen_task) * nb_ports * nb_txq, 0);
+	if (!pktgen_tx_tasks)
+		RTE_LOG(ERR, USER1, "out of memory?\n");
+	if (!pktgen_rx_tasks)
+		pktgen_rx_tasks = rte_malloc(NULL,
+			sizeof(struct pktgen_task) * nb_ports * nb_rxq, 0);
+	if (!pktgen_rx_tasks)
+		RTE_LOG(ERR, USER1, "out of memory?\n");
+}
+
+static void
+pktgen_end(portid_t pi __rte_unused)
+{
+	if (pktgen_tx_tasks)
+		rte_free(pktgen_tx_tasks);
+	pktgen_tx_tasks = NULL;
+	if (pktgen_rx_tasks)
+		rte_free(pktgen_rx_tasks);
+	pktgen_rx_tasks = NULL;
+}
+
+struct fwd_engine pktgen_engine = {
+	.fwd_mode_name  = "pktgen",
+	.port_fwd_begin = pktgen_begin,
+	.port_fwd_end   = pktgen_end,
+	.packet_fwd     = pg_fwd,
+};
+
+/********************************************************/
+/* Control thread functions                             */
+/********************************************************/
+
+/* "pktgen loopback" command */
+struct cmd_pktgen_cmd_result {
+	cmdline_fixed_string_t pktgen;
+	cmdline_fixed_string_t cmd;
+	uint8_t mode;
+};
+
+cmdline_parse_token_string_t cmd_pktgen_cmd_pktgen =
+	TOKEN_STRING_INITIALIZER(struct cmd_pktgen_cmd_result, pktgen, "pktgen");
+cmdline_parse_token_string_t cmd_pktgen_cmd_cmd =
+	TOKEN_STRING_INITIALIZER(struct cmd_pktgen_cmd_result, cmd, "idle");
+cmdline_parse_token_string_t cmd_pktgen_cmd_mode =
+		TOKEN_NUM_INITIALIZER(struct cmd_pktgen_cmd_result, mode, UINT8);
+
+static void
+cmd_pktgen_cmd_parsed(
+	void *parsed_result,
+	__attribute__((unused)) struct cmdline *cl,
+	__attribute__((unused)) void *data)
+{
+	struct cmd_pktgen_cmd_result *res = parsed_result;
+
+	pktgen_idle_mode = res->mode;
+	printf("PktGen idle mode: %hhu\n", res->mode);
+}
+
+cmdline_parse_inst_t cmd_pktgen_cmd = {
+	.f = cmd_pktgen_cmd_parsed,
+	.data = NULL,
+	.help_str = "pktgen idle <mode>: 0-drop 1-loopback 2-forward 3-switch",
+	.tokens = {
+		(void *)&cmd_pktgen_cmd_pktgen,
+		(void *)&cmd_pktgen_cmd_cmd,
+		(void *)&cmd_pktgen_cmd_mode,
+		NULL,
+	},
+};
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index c3ab44849..2a50e522b 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -159,6 +159,7 @@ struct fwd_engine * fwd_engines[] = {
 	&mac_fwd_engine,
 	&mac_swap_engine,
 	&flow_gen_engine,
+	&pktgen_engine,
 	&rx_only_engine,
 	&tx_only_engine,
 	&csum_fwd_engine,
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 1639d27e7..ec83c212d 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -296,6 +296,7 @@ extern struct fwd_engine io_fwd_engine;
 extern struct fwd_engine mac_fwd_engine;
 extern struct fwd_engine mac_swap_engine;
 extern struct fwd_engine flow_gen_engine;
+extern struct fwd_engine pktgen_engine;
 extern struct fwd_engine rx_only_engine;
 extern struct fwd_engine tx_only_engine;
 extern struct fwd_engine csum_fwd_engine;
-- 
2.13.3

[dpdk-dev] [RFC v1 7/9] app/testpmd: add pktgen engine scapy commands

[Xueming Li]

new commands:
tx: send packets
rx: receive packets
expect: send and expect back

Signed-off-by: Xueming Li <xuemingl@mellanox.com>

Conflicts:
	app/test-pmd/Makefile
	app/test-pmd/testpmd.h
---
 app/test-pmd/Makefile  |   5 +
 app/test-pmd/cmdline.c |  14 +-
 app/test-pmd/pktgen.c  | 493 ++++++++++++++++++++++++++++++++++++++++++++++++-
 app/test-pmd/testpmd.h |   4 +
 4 files changed, 512 insertions(+), 4 deletions(-)

diff --git a/app/test-pmd/Makefile b/app/test-pmd/Makefile
index 2610a8d49..462f97418 100644
--- a/app/test-pmd/Makefile
+++ b/app/test-pmd/Makefile
@@ -65,6 +65,11 @@ SRCS-$(CONFIG_RTE_LIBRTE_IEEE1588) += ieee1588fwd.c
 ifeq ($(CONFIG_RTE_LIBRTE_PMD_SOFTNIC)$(CONFIG_RTE_LIBRTE_SCHED),yy)
 SRCS-y += tm.c
 endif
+ifeq ($(CONFIG_RTE_LIBRTE_PYTHON),y)
+LDLIBS += -l$(CONFIG_RTE_LIBRTE_PYTHON_VERSION)
+CFLAGS += -I/usr/include/$(CONFIG_RTE_LIBRTE_PYTHON_VERSION)
+endif
+
 
 ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y)
 
diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index d0eb00c1f..b40fe1ac7 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -105,7 +105,7 @@
 #include "cmdline_mtr.h"
 #include "cmdline_tm.h"
 
-static struct cmdline *testpmd_cl;
+struct cmdline *testpmd_cl;
 
 static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue);
 
@@ -15595,6 +15595,12 @@ cmdline_parse_inst_t cmd_py_run = {
 /* ******************************************************************************** */
 
 extern cmdline_parse_inst_t cmd_pktgen_cmd;
+extern cmdline_parse_inst_t cmd_expect_short;
+extern cmdline_parse_inst_t cmd_expect;
+extern cmdline_parse_inst_t cmd_tx;
+extern cmdline_parse_inst_t cmd_tx_short;
+extern cmdline_parse_inst_t cmd_rx;
+extern cmdline_parse_inst_t cmd_rx_short;
 
 /* list of instructions */
 cmdline_parse_ctx_t main_ctx[] = {
@@ -15604,6 +15610,12 @@ cmdline_parse_ctx_t main_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_load_from_file,
 #ifdef RTE_LIBRTE_PYTHON
 	(cmdline_parse_inst_t *)&cmd_py_run,
+	(cmdline_parse_inst_t *)&cmd_expect_short,
+	(cmdline_parse_inst_t *)&cmd_expect,
+	(cmdline_parse_inst_t *)&cmd_tx_short,
+	(cmdline_parse_inst_t *)&cmd_tx,
+	(cmdline_parse_inst_t *)&cmd_rx_short,
+	(cmdline_parse_inst_t *)&cmd_rx,
 #endif
 	(cmdline_parse_inst_t *)&cmd_pktgen_cmd,
 	(cmdline_parse_inst_t *)&cmd_showport,
diff --git a/app/test-pmd/pktgen.c b/app/test-pmd/pktgen.c
index f4a1b00a9..423459b37 100644
--- a/app/test-pmd/pktgen.c
+++ b/app/test-pmd/pktgen.c
@@ -83,7 +83,6 @@
 #define PKTGEN_ROUND_END 0x1
 #define PKTGEN_TASK_END 0x0
 #define PKTGEN_TASK_START 0x3
-
 struct pktgen_task_stats{
 	uint8_t active;
 	uint16_t round;		/* number txrx */
@@ -566,6 +565,495 @@ struct fwd_engine pktgen_engine = {
 /* Control thread functions                             */
 /********************************************************/
 
+#ifdef RTE_LIBRTE_PYTHON
+
+#define US_TSC(us) ((us) * (rte_get_timer_hz() / 1000000L));
+#define TSC_US(tsc) ((tsc) * 1e6 / rte_get_timer_hz())
+
+struct cmd_pktgen_cmd {
+	cmdline_fixed_string_t cmd;
+	cmdline_fixed_string_t pattern;
+	portid_t port;
+	uint64_t count;
+	uint64_t round;
+	uint16_t timeout; /* unit: ms */
+	uint16_t verbose;
+	struct pktgen_task_stats stats;
+};
+
+struct cmd_pg_txrx_cmd {
+	cmdline_fixed_string_t expect;
+	struct cmd_pktgen_cmd tx;
+	struct cmd_pktgen_cmd rx;
+	cmdline_fixed_string_t field;
+	uint64_t val;
+};
+
+/*
+ * get min/max time and count sum
+ */
+static void
+cmd_pg_port_poll(portid_t port, struct pktgen_task_stats *sum, int tx)
+{
+	struct pktgen_task* task;
+	queueid_t i, n;
+
+	n = tx ? nb_txq : nb_rxq;
+	for (i = 0, sum->count = 0; i < n; i++) {
+		task = tx ? task_tx(port, i) : task_rx(port, i);
+		sum->count += task->stats.count;
+		if (task->stats.round > sum->round)
+			sum->round = task->stats.round;
+		if (task->stats.start) {
+			if (sum->start && task->stats.start < sum->start)
+				sum->start = task->stats.start;
+			if (!sum->start)
+				sum->start = task->stats.start;
+			if (sum->end < task->stats.end)
+				sum->end = task->stats.end;
+		}
+	}
+}
+
+static int
+cmd_pg_scapy_to_mbuf(char *scapy, portid_t port, struct pktgen_task *task)
+{
+	int socket;
+	struct rte_mempool *pool;
+
+	socket = port_numa[port];
+	if (socket == NUMA_NO_CONFIG)
+		socket = ports[port].socket_id;
+	pool = mbuf_pool_find(socket);
+	task->data = rte_python_scapy_to_mbufs(pool, scapy, &task->cnt_mbufs);
+	return !task->data;
+}
+
+static inline int
+cmd_pg_init(void)
+{
+	if (rte_python_init())
+		return -1;
+	if (pktgen_idle_mode != 0) {
+		pg_idle_set(0);
+		rte_delay_ms(1);
+	}
+	if (cur_fwd_eng != &pktgen_engine) {
+		set_pkt_forwarding_mode(pktgen_engine.fwd_mode_name);
+		if (!test_done)
+			stop_packet_forwarding();
+	}
+	if (test_done)
+		start_packet_forwarding(0);
+	/* reset task memory */
+	RTE_ASSERT(pktgen_tx_tasks && pktgen_rx_tasks);
+	memset(pktgen_tx_tasks, 0,
+			nb_ports * nb_txq * sizeof(struct pktgen_task));
+	memset(pktgen_rx_tasks, 0,
+			nb_ports * nb_rxq * sizeof(struct pktgen_task));
+	return 0;
+}
+
+static void
+cmd_pg_cleanup(struct cmd_pg_txrx_cmd *cmd)
+{
+	struct pktgen_task *task;
+	queueid_t q;
+	uint16_t m;
+
+	RTE_ASSERT(pktgen_tx_tasks && pktgen_rx_tasks);
+	/* free all tx queue mbufs */
+	for (q = 0; cmd->tx.count && q < nb_txq; q++) {
+		task = task_tx(cmd->tx.port, q);
+		if (!task->data)
+			continue;
+		m = task->cnt_mbufs;
+		while(m)
+			rte_pktmbuf_free(pg_task_template_get(task, --m));
+		task->data = NULL;
+	}
+}
+
+static void
+cmd_pg_rx(struct cmd_pg_txrx_cmd *cmd)
+{
+	struct pktgen_task *task;
+	int i;
+
+	RTE_ASSERT(cmd);
+	memset(&cmd->rx.stats, 0, sizeof(cmd->rx.stats));
+	for (i = 0; i < nb_rxq; i++) {
+		task = task_rx(cmd->rx.port, i);
+		if (cmd->tx.count)
+			task->data = task_tx(cmd->tx.port, 0);
+		task->count = cmd->rx.count;
+		task->round = cmd->rx.round;
+		task->verbose = cmd->rx.verbose & 0xff;
+		if (cmd->field && strlen(cmd->field)) {
+				task->field = cmd->field;
+				task->val = cmd->val;
+		}
+		task->stats.active = PKTGEN_TASK_START;
+	}
+}
+
+static int
+cmd_pg_tx(struct cmd_pktgen_cmd* cmd, int txrx)
+{
+	struct pktgen_task *task;
+
+	RTE_ASSERT(cmd);
+	task = task_tx(cmd->port, 0);
+	if (cmd_pg_scapy_to_mbuf(cmd->pattern, cmd->port, task))
+		return -1;
+	/* send out using queue 0 */
+	memset(&cmd->stats, 0, sizeof(cmd->stats));
+	if (cmd->count == UINT64_MAX)
+		cmd->count = task->cnt_mbufs;
+	task->count = cmd->count;
+	task->round = cmd->round;
+	task->verbose = cmd->verbose & 0xff;
+	task->txrx = txrx;
+	task->stats.active = PKTGEN_TASK_START;
+	return 0;
+}
+
+static void
+cmd_pg_wait(struct cmdline *cl, struct cmd_pktgen_cmd* cmd,
+		uint64_t timeout, int tx)
+{
+	char c = 0;
+	int flags;
+
+	flags = fcntl(cl->s_in, F_GETFL, 0);
+	RTE_ASSERT(flags >= 0);
+	fcntl(cl->s_in, F_SETFL, flags | O_NONBLOCK);
+	memset(&cmd->stats, 0, sizeof(cmd->stats));
+	while (1) {
+		cmd_pg_port_poll(cmd->port, &cmd->stats, tx);
+		if (cmd->count && cmd->round == cmd->stats.round)
+			break;
+		if (timeout && rte_rdtsc() > timeout)
+			break;
+		/* detect ctrl+c or ctrl+d */
+		if (!timeout && read(cl->s_in, &c, 1) && (c == 3 || c == 4))
+			break;
+		rte_delay_ms(1);
+	}
+	if (!cmd->stats.end)
+		cmd->stats.end = rte_rdtsc();
+	fcntl(cl->s_in, F_SETFL, flags);
+}
+
+/* expect command */
+cmdline_parse_token_string_t cmd_expect_cmd =
+	TOKEN_STRING_INITIALIZER(struct cmd_pg_txrx_cmd, expect, "expect");
+cmdline_parse_token_num_t cmd_expect_tx_port =
+	TOKEN_NUM_INITIALIZER(struct cmd_pg_txrx_cmd, tx.port, UINT16);
+cmdline_parse_token_num_t cmd_expect_rx_port =
+	TOKEN_NUM_INITIALIZER(struct cmd_pg_txrx_cmd, rx.port, UINT16);
+cmdline_parse_token_string_t cmd_expect_pattern =
+	TOKEN_STRING_INITIALIZER(struct cmd_pg_txrx_cmd, tx.pattern, NULL);
+cmdline_parse_token_num_t cmd_expect_count =
+	TOKEN_NUM_INITIALIZER(struct cmd_pg_txrx_cmd, tx.count, UINT64);
+cmdline_parse_token_num_t cmd_expect_round =
+	TOKEN_NUM_INITIALIZER(struct cmd_pg_txrx_cmd, tx.round, UINT64);
+cmdline_parse_token_num_t cmd_expect_rx_timeout =
+	TOKEN_NUM_INITIALIZER(struct cmd_pg_txrx_cmd, rx.timeout, UINT64);
+cmdline_parse_token_num_t cmd_expect_verbose =
+	TOKEN_NUM_INITIALIZER(struct cmd_pg_txrx_cmd, tx.verbose, UINT16);
+cmdline_parse_token_string_t cmd_expect_field =
+	TOKEN_STRING_INITIALIZER(struct cmd_pg_txrx_cmd, field, NULL);
+cmdline_parse_token_num_t cmd_expect_val =
+	TOKEN_NUM_INITIALIZER(struct cmd_pg_txrx_cmd, val, UINT64);
+
+static void
+cmd_expect_parsed(
+	void *parsed_result,
+	__attribute__((unused)) struct cmdline *cl,
+	__attribute__((unused)) void *data)
+{
+	struct cmd_pg_txrx_cmd *cmd = parsed_result;
+	struct pktgen_task_stats *stats_tx = &cmd->tx.stats;
+	struct pktgen_task_stats *stats_rx = &cmd->rx.stats;
+	uint64_t timeout = 0;
+
+	if (port_id_is_invalid(cmd->tx.port, ENABLED_WARN) ||
+		port_id_is_invalid(cmd->rx.port, ENABLED_WARN))
+		return;
+	if (cmd_pg_init())
+		return;
+	if (!strcmp(cmd->field, "non") || !strcmp(cmd->field, "0"))
+		cmd->field[0] = 0;
+	cmd->rx.verbose = cmd->tx.verbose & 0xff;
+	cmd->tx.verbose = cmd->tx.verbose >> 8;
+	cmd->rx.round = cmd->tx.round;
+	cmd->rx.count = cmd->tx.count;
+	if (cmd->tx.count == 0)
+		cmd->tx.count = UINT64_MAX;
+	/* prepare task */
+	if (cmd_pg_tx(&cmd->tx, 1))
+		return;
+	if (cmd->rx.count == UINT64_MAX)
+		cmd->rx.count = cmd->tx.count;
+	cmd_pg_rx(cmd);
+	/* wait task */
+	if (cmd->rx.timeout)
+		timeout = rte_rdtsc() + US_TSC(cmd->rx.timeout * 1000) ;
+	pktgen_busy = 1;
+	cmd_pg_wait(cl, &cmd->tx, timeout, 1);
+	cmd_pg_wait(cl, &cmd->rx, timeout, 0);
+	pktgen_busy = 0;
+	/* print stats */
+	float t_tx = TSC_US(stats_tx->end - stats_tx->start);
+	float t_rx = TSC_US(stats_rx->end - stats_rx->start);
+	float t_ttl = TSC_US(RTE_MAX(stats_rx->end, stats_tx->end) -
+			RTE_MIN(stats_rx->start, stats_tx->start));
+	if (cmd->rx.count == 0)
+		stats_rx->round = 1;
+	int failed = (stats_tx->round != cmd->tx.round ||
+			stats_rx->round != cmd->rx.round ||
+			stats_rx->count != cmd->rx.count);
+	if (stats_tx->round == 0)
+		stats_tx->round = 1;
+	if (stats_rx->round == 0)
+		stats_rx->round = 1;
+	uint64_t nb_tx = stats_tx->count + cmd->tx.count * (stats_tx->round - 1);
+	uint64_t nb_rx = stats_rx->count + cmd->rx.count * (stats_rx->round - 1);
+	if (failed || !(verbose_level & 0x40)) /* mute */
+		printf("%s"
+			"tx: %lu/%lu %.3fus %fmpps"
+			"\trx: %lu/%lu %.3fus %fmpps"
+			"\tround: %u/%lu %.3fus"
+			"\ttotal: %.3fus %fmpps\n",
+			failed ? "Failed " : "",
+			nb_tx, cmd->tx.count * cmd->tx.round, t_tx, nb_tx/t_tx,
+			nb_rx, cmd->rx.count * cmd->rx.round, t_rx, nb_rx/t_rx,
+			stats_rx->round, cmd->rx.round, t_ttl / stats_rx->round,
+			t_ttl, nb_rx / t_ttl);
+	/* clean up */
+	cmd_pg_cleanup(cmd);
+}
+
+cmdline_parse_inst_t cmd_expect = {
+	.f = cmd_expect_parsed,
+	.data = NULL,
+	.help_str = "expect <tx_port> <rx_port> <scapy> <count> <round> <timeout(ms)> <verbose> <field> <val>\n"
+			"\t\t\tSend packet and expecting same back",
+	.tokens = {
+		(void *)&cmd_expect_cmd,
+		(void *)&cmd_expect_tx_port,
+		(void *)&cmd_expect_rx_port,
+		(void *)&cmd_expect_pattern,
+		(void *)&cmd_expect_count,
+		(void *)&cmd_expect_round,
+		(void *)&cmd_expect_rx_timeout,
+		(void *)&cmd_expect_verbose,
+		(void *)&cmd_expect_field,
+		(void *)&cmd_expect_val,
+		NULL,
+	},
+};
+
+static void
+cmd_expect_short_parsed(
+	void *parsed_result,
+	__attribute__((unused)) struct cmdline *cl,
+	__attribute__((unused)) void *data)
+{
+	struct cmd_pg_txrx_cmd *res = parsed_result;
+
+	/* memory not clean, have to clear unused fields */
+	res->tx.round = 1;
+	res->tx.count = UINT64_MAX; /* detect from pattern */
+	res->tx.verbose = verbose_level;
+	res->rx.timeout = 20;
+	res->rx.verbose = verbose_level & 0xff;
+	res->field[0] = 0;
+	cmd_expect_parsed(res, cl, data);
+
+}
+
+cmdline_parse_inst_t cmd_expect_short = {
+	.f = cmd_expect_short_parsed,
+	.data = NULL,
+	.help_str = "expect <tx_port> <rx_port> <scapy>: tx 1 and expect 1",
+	.tokens = {
+		(void *)&cmd_expect_cmd,
+		(void *)&cmd_expect_tx_port,
+		(void *)&cmd_expect_rx_port,
+		(void *)&cmd_expect_pattern,
+		NULL,
+	},
+};
+
+/* tx command */
+
+cmdline_parse_token_string_t cmd_tx_cmd =
+	TOKEN_STRING_INITIALIZER(struct cmd_pg_txrx_cmd, tx.cmd, "tx");
+cmdline_parse_token_string_t cmd_tx_pattern =
+	TOKEN_STRING_INITIALIZER(struct cmd_pg_txrx_cmd, tx.pattern, NULL);
+cmdline_parse_token_num_t cmd_tx_port =
+	TOKEN_NUM_INITIALIZER(struct cmd_pg_txrx_cmd, tx.port, UINT16);
+cmdline_parse_token_num_t cmd_tx_count =
+	TOKEN_NUM_INITIALIZER(struct cmd_pg_txrx_cmd, tx.count, UINT64);
+cmdline_parse_token_num_t cmd_tx_verbose =
+	TOKEN_NUM_INITIALIZER(struct cmd_pg_txrx_cmd, tx.verbose, UINT16);
+
+static void
+cmd_tx_parsed(
+	void *parsed_result,
+	__attribute__((unused)) struct cmdline *cl,
+	__attribute__((unused)) void *data)
+{
+	struct cmd_pg_txrx_cmd *cmd = parsed_result;
+
+	if (port_id_is_invalid(cmd->tx.port, ENABLED_WARN))
+		return;
+	if (cmd_pg_init())
+		return;
+	memset(&cmd->rx, 0, sizeof(cmd->rx));
+	cmd->tx.round = 1;
+	if (cmd_pg_tx(&cmd->tx, 0))
+		return;
+	pktgen_busy = 1;
+	cmd_pg_wait(cl, &cmd->tx, 0, 1);
+	pktgen_busy = 0;
+	double t = TSC_US(cmd->tx.stats.end - cmd->tx.stats.start);
+	printf("%s%lu/%lu packets sent in %.3fus %fmpps\n",
+			cmd->tx.count && cmd->tx.stats.count != cmd->tx.count ?
+					"Failed: " : "",
+			cmd->tx.stats.count, cmd->tx.count, t,
+			cmd->tx.stats.count / t);
+	cmd_pg_cleanup(cmd);
+}
+
+cmdline_parse_inst_t cmd_tx = {
+	.f = cmd_tx_parsed,
+	.data = NULL,
+	.help_str = "tx <port> <scapy> <count> <verbose>",
+	.tokens = {
+		(void *)&cmd_tx_cmd,
+		(void *)&cmd_tx_port,
+		(void *)&cmd_tx_pattern,
+		(void *)&cmd_tx_count,
+		(void *)&cmd_tx_verbose,
+		NULL,
+	},
+};
+
+static void
+cmd_tx_short_parsed(
+	void *parsed_result,
+	__attribute__((unused)) struct cmdline *cl,
+	__attribute__((unused)) void *data)
+{
+	struct cmd_pg_txrx_cmd *cmd = parsed_result;
+
+	cmd->tx.count = 0;
+	cmd->tx.verbose = verbose_level >> 8;
+	cmd_tx_parsed(cmd, cl, data);
+}
+
+cmdline_parse_inst_t cmd_tx_short = {
+	.f = cmd_tx_short_parsed,
+	.data = NULL,
+	.help_str = "tx <port> <scapy>: tx 0 Ether()/IP()/UDP()",
+	.tokens = {
+		(void *)&cmd_tx_cmd,
+		(void *)&cmd_tx_port,
+		(void *)&cmd_tx_pattern,
+		NULL,
+	},
+};
+
+/* rx command */
+cmdline_parse_token_string_t cmd_rx_cmd =
+	TOKEN_STRING_INITIALIZER(struct cmd_pg_txrx_cmd, rx, "rx");
+cmdline_parse_token_num_t cmd_rx_port =
+	TOKEN_NUM_INITIALIZER(struct cmd_pg_txrx_cmd, rx.port, UINT16);
+cmdline_parse_token_num_t cmd_rx_count =
+	TOKEN_NUM_INITIALIZER(struct cmd_pg_txrx_cmd, rx.count, UINT64);
+cmdline_parse_token_num_t cmd_rx_timeout =
+	TOKEN_NUM_INITIALIZER(struct cmd_pg_txrx_cmd, rx.timeout, UINT16);
+cmdline_parse_token_num_t cmd_rx_verbose =
+	TOKEN_NUM_INITIALIZER(struct cmd_pg_txrx_cmd, rx.verbose, UINT16);
+
+/* Common result structure for rx commands */
+static void
+cmd_rx_parsed(
+	void *parsed_result,
+	__attribute__((unused)) struct cmdline *cl,
+	__attribute__((unused)) void *data)
+{
+	struct cmd_pg_txrx_cmd *cmd = parsed_result;
+	uint64_t timeout = 0;
+
+	if (port_id_is_invalid(cmd->rx.port, ENABLED_WARN))
+		return;
+	if (cmd_pg_init())
+		return;
+	memset(&cmd->tx, 0, sizeof(cmd->tx));
+	cmd->field[0] = 0;
+	cmd->rx.round = 1;
+	cmd_pg_rx(cmd);
+	if (cmd->rx.timeout)
+		timeout =  rte_rdtsc() + US_TSC(cmd->rx.timeout * 1e6);
+	pktgen_busy = 1;
+	cmd_pg_wait(cl, &cmd->rx, timeout, 0);
+	pktgen_busy = 0;
+	/* print stats */
+	float t = TSC_US(cmd->rx.stats.end - cmd->rx.stats.start);
+	printf("%s%lu/%lu packets received in %.3fus %fmpps\n",
+			cmd->rx.count && cmd->rx.stats.count != cmd->rx.count ?
+					"Failed: " : "",
+			cmd->rx.stats.count, cmd->rx.count, t,
+			t ? cmd->rx.stats.count / t : 0);
+	/* clean up */
+	cmd_pg_cleanup(cmd);
+}
+
+cmdline_parse_inst_t cmd_rx = {
+	.f = cmd_rx_parsed,
+	.data = NULL,
+	.help_str = "rx <port> <count> <timeout(s)> <verbose>",
+	.tokens = {
+		(void *)&cmd_rx_cmd,
+		(void *)&cmd_rx_port,
+		(void *)&cmd_rx_count,
+		(void *)&cmd_rx_timeout,
+		(void *)&cmd_rx_verbose,
+		NULL,
+	},
+};
+
+static void
+cmd_rx_short_parsed(
+	void *parsed_result,
+	__attribute__((unused)) struct cmdline *cl,
+	__attribute__((unused)) void *data)
+{
+	struct cmd_pg_txrx_cmd *cmd = parsed_result;
+
+	cmd->rx.count = 0; /* endless */
+	cmd->rx.timeout = 0; /* endless */
+	cmd->rx.verbose = verbose_level;
+	cmd_rx_parsed(cmd, cl, data);
+}
+
+cmdline_parse_inst_t cmd_rx_short = {
+	.f = cmd_rx_short_parsed,
+	.data = NULL,
+	.help_str = "rx <port>",
+	.tokens = {
+		(void *)&cmd_rx_cmd,
+		(void *)&cmd_rx_port,
+		NULL,
+	},
+};
+
+#endif
+
 /* "pktgen loopback" command */
 struct cmd_pktgen_cmd_result {
 	cmdline_fixed_string_t pktgen;
@@ -588,8 +1076,7 @@ cmd_pktgen_cmd_parsed(
 {
 	struct cmd_pktgen_cmd_result *res = parsed_result;
 
-	pktgen_idle_mode = res->mode;
-	printf("PktGen idle mode: %hhu\n", res->mode);
+	pg_idle_set(res->mode);
 }
 
 cmdline_parse_inst_t cmd_pktgen_cmd = {
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index ec83c212d..daa5d4e0c 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -34,6 +34,8 @@
 #ifndef _TESTPMD_H_
 #define _TESTPMD_H_
 
+#include <cmdline_parse.h>
+
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
 #include <rte_gro.h>
@@ -365,6 +367,8 @@ extern uint8_t flow_isolate_all; /**< set by "--flow-isolate-all */
 extern uint8_t  mp_anon; /**< set by "--mp-anon" parameter */
 extern uint8_t no_link_check; /**<set by "--disable-link-check" parameter */
 extern volatile int test_done; /* stop packet forwarding when set to 1. */
+extern cmdline_parse_ctx_t main_ctx[]; /* command line parser */
+extern struct cmdline *testpmd_cl;
 extern uint8_t lsc_interrupt; /**< disabled by "--no-lsc-interrupt" parameter */
 extern uint8_t rmv_interrupt; /**< disabled by "--no-rmv-interrupt" parameter */
 extern uint32_t event_print_mask;
-- 
2.13.3

[dpdk-dev] [RFC v1 8/9] test/expect: add expect test scripts

[Xueming Li]

Please enable python module to run thesse scripts
---
 test/expect/init.exp |  28 +++++++++++
 test/expect/rx.exp   | 134 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 162 insertions(+)
 create mode 100644 test/expect/init.exp
 create mode 100644 test/expect/rx.exp

diff --git a/test/expect/init.exp b/test/expect/init.exp
new file mode 100644
index 000000000..475ff39f9
--- /dev/null
+++ b/test/expect/init.exp
@@ -0,0 +1,28 @@
+# scapy vars
+py eth = Ether(src="00:11:22:33:44:55",dst="aa:bb:cc:dd:ee:ff")
+py ethb = Ether(src="00:11:22:33:44:55",dst="ff:ff:ff:ff:ff:ff")
+py ethm = Ether(src="00:11:22:33:44:55",dst="01:02:03:04:05:06")
+py ip = IP(src="4.3.2.1",dst="44.33.22.11")
+py ipc = IP(src="4.3.2.1",dst="44.33.22.11",chksum=1)
+py ip1 = IP(src="4.3.2.1",dst="44.33.22.12")
+py ip2 = IP(src="4.3.2.2",dst="44.33.22.11")
+py ipv6 = IPv6(src="::2",dst="::1")
+py ipv6c = IPv6(src="::2",dst="::1")
+py ipv61 = IPv6(src="::2",dst="::3")
+py ipv62 = IPv6(src="::3",dst="::1")
+py udp = UDP(sport=54321,dport=4789)
+py udpc = UDP(sport=54321,dport=4789,chksum=1)
+py udp1 = UDP(sport=54322,dport=4789)
+py udp2 = UDP(sport=54321,dport=4790)
+py tcp = TCP(sport=54321,dport=4789)
+py tcpc = TCP(sport=54321,dport=4789,chksum=1)
+py tcp1 = TCP(sport=54322,dport=4789)
+py tcp2 = TCP(sport=54321,dport=4790)
+py vxlan = VXLAN(vni=0x12345)
+py l3vxlan = VXLAN(vni=0x12345)
+py gre = GRE()
+py grex = GRE(chksum_present=1,key_present=1,seqnum_present=1,version=1,chksum=0xbad0,key=0x12345678,seqence_number=0x87654321)
+py x4 = eth/ip/udp/vxlan
+py x6 = eth/ipv6/udp/vxlan
+py g4 = eth/ip/gre
+py g6 = eth/ipv6/gre
diff --git a/test/expect/rx.exp b/test/expect/rx.exp
new file mode 100644
index 000000000..a67f016af
--- /dev/null
+++ b/test/expect/rx.exp
@@ -0,0 +1,134 @@
+# sudo build/app/testpmd -l4-8   -m 200 -w 82:0.0,tx_vec_en=0,rx_vec_en=0 -- -i -a --total-num-mbufs=4096 --txq=4 --rxq=4 --nb-cores=4 --forward-mode=pktgen --rss-udp --enable-rx-cksum
+# sudo build/app/testpmd -l4-8   -m 200 -w 82:0.0,tx_vec_en=1,rx_vec_en=1 -- -i -a --total-num-mbufs=4096 --txq=4 --rxq=4 --nb-cores=4 --forward-mode=pktgen --rss-udp --enable-rx-cksum
+
+
+load  test/expect/init.exp
+
+# show loaded CLIs, mute expect summary info
+# set  verbose  0x8040
+
+port config all rss all
+stop
+port stop 0
+port start 0
+start
+expect 0 0 eth/ip/udp 1 1 20 0x0 queue 1
+expect 0 0 eth/ip/udp 1 1 20 0x0 ptype 0x291
+expect 0 0 eth/ip/udp 1 1 20 0x0 ol_flags 0x180
+expect 0 0 eth/ip/udp 1 1 20 0x0 rss 0xf4ff21c5
+expect 0 0 eth/ip1/udp 1 1 20 0x0 rss 0xa1290ec
+expect 0 0 eth/ip2/udp 1 1 20 0x0 rss 0xfe42683
+expect 0 0 eth/ip/udp1 1 1 20 0x0 rss 0x15b5ba8e
+expect 0 0 eth/ip/udp2 1 1 20 0x0 rss 0x6fb4b12a
+
+
+expect 0 0 eth/ip/udpc 1 1 20 0x0 ol_flags 0x80
+expect 0 0 eth/ip/udpc 1 1 20 0x0 ptype 0x291
+expect 0 0 eth/ipc/udp 1 1 20 0x0 ol_flags 0x100
+expect 0 0 eth/ipc/udp 1 1 20 0x0 ptype 0x291
+expect 0 0 eth/ipc/udpc 1 1 20 0x0 ol_flags 0x0
+expect 0 0 eth/ipc/udpc 1 1 20 0x0 ptype 0x291
+expect 0 0 eth/ip/tcp 1 1 20 0x0 ol_flags 0x180
+expect 0 0 eth/ip/tcp 1 1 20 0x0 ptype 0x191
+expect 0 0 eth/ip/tcp 1 1 20 0x0 queue 1
+expect 0 0 eth/ip/tcp 1 1 20 0x0 rss 0xf4ff21c5
+expect 0 0 eth/ip1/tcp 1 1 20 0x0 rss 0xa1290ec
+expect 0 0 eth/ip2/tcp 1 1 20 0x0 rss 0xfe42683
+expect 0 0 eth/ip/tcp1 1 1 20 0x0 rss 0x15b5ba8e
+expect 0 0 eth/ip/tcp2 1 1 20 0x0 rss 0x6fb4b12a
+
+expect 0 0 eth/ip/tcpc 1 1 20 0x0 ol_flags 0x80
+expect 0 0 eth/ip/tcpc 1 1 20 0x0 ptype 0x191
+expect 0 0 eth/ipc/tcp 1 1 20 0x0 ol_flags 0x100
+expect 0 0 eth/ipc/tcp 1 1 20 0x0 ptype 0x191
+expect 0 0 eth/ipc/tcpc 1 1 20 0x0 ol_flags 0x0
+expect 0 0 eth/ipc/tcpc 1 1 20 0x0 ptype 0x191
+expect 0 0 eth/ip 1 1 20 0x0 queue 1
+expect 0 0 eth/ip 1 1 20 0x0 ptype 0x691
+expect 0 0 eth/ip 1 1 20 0x0 ol_flags 0x80
+expect 0 0 eth/ip 1 1 20 0x0 rss 0xe7fd7ed5
+expect 0 0 eth/ip1 1 1 20 0x0 rss 0x1910cffc
+expect 0 0 eth/ip2 1 1 20 0x0 rss 0x1ce67993
+
+expect 0 0 eth/ipc 1 1 20 0x0 ol_flags 0x0
+expect 0 0 eth/ipc 1 1 20 0x0 ptype 0x691
+expect 0 0 eth/fragment(ip/udp/("a"*100),64)[0] 1 1 20 0x0 queue 1
+expect 0 0 eth/fragment(ip/udp/("a"*100),64)[0] 1 1 20 0x0 ptype 0x391
+expect 0 0 eth/fragment(ip/udp/("a"*100),64)[0] 1 1 20 0x0 ol_flags 0x80
+expect 0 0 eth/fragment(ip/udp/("a"*100),64)[0] 1 1 20 0x0 rss 0xe7fd7ed5
+expect 0 0 eth/fragment(ip/udp/("a"*100),64)[1] 1 1 20 0x0 queue 1
+expect 0 0 eth/fragment(ip/udp/("a"*100),64)[1] 1 1 20 0x0 ptype 0x391
+expect 0 0 eth/fragment(ip/udp/("a"*100),64)[1] 1 1 20 0x0 ol_flags 0x80
+expect 0 0 eth/fragment(ip/udp/("a"*100),64)[1] 1 1 20 0x0 rss 0xe7fd7ed5
+expect 0 0 eth/ipv6/udp 1 1 20 0x0 rss 0x1799a1f0
+expect 0 0 eth/ipv6/udp 1 1 20 0x0 ptype 0x2e1
+expect 0 0 eth/ipv6/udp 1 1 20 0x0 ol_flags 0x180
+expect 0 0 eth/ipv6/udp 1 1 20 0x0 rss 0x1799a1f0
+expect 0 0 eth/ipv61/udp 1 1 20 0x0 rss 0x5616a88c
+expect 0 0 eth/ipv62/udp 1 1 20 0x0 rss 0x2256bd9c
+expect 0 0 eth/ipv6/udp1 1 1 20 0x0 rss 0xc1ca0f8
+expect 0 0 eth/ipv6/udp2 1 1 20 0x0 rss 0x1691b8ef
+
+
+expect 0 0 eth/ipv6/udpc 1 1 20 0x0 ol_flags 0x80
+expect 0 0 eth/ipv6/tcp 1 1 20 0x0 ol_flags 0x180
+expect 0 0 eth/ipv6/tcp 1 1 20 0x0 ptype 0x1e1
+expect 0 0 eth/ipv6/tcp 1 1 20 0x0 queue 0
+expect 0 0 eth/ipv6/tcp 1 1 20 0x0 rss 0x1799a1f0
+expect 0 0 eth/ipv61/tcp 1 1 20 0x0 rss 0x5616a88c
+expect 0 0 eth/ipv62/tcp 1 1 20 0x0 rss 0x2256bd9c
+expect 0 0 eth/ipv6/tcp1 1 1 20 0x0 rss 0xc1ca0f8
+expect 0 0 eth/ipv6/tcp2 1 1 20 0x0 rss 0x1691b8ef
+
+expect 0 0 eth/ipv6/tcpc 1 1 20 0x0 ol_flags 0x80
+expect 0 0 eth/ipv6 1 1 20 0x0 queue 3
+expect 0 0 eth/ipv6 1 1 20 0x0 ptype 0x6e1
+expect 0 0 eth/ipv6 1 1 20 0x0 ol_flags 0x80
+expect 0 0 eth/ipv6 1 1 20 0x0 rss 0x99f99ccf
+expect 0 0 eth/ipv61 1 1 20 0x0 rss 0xd87695b3
+expect 0 0 eth/ipv62 1 1 20 0x0 rss 0xac3680a3
+
+expect 0 0 eth/fragment6(ipv6/IPv6ExtHdrFragment()/udp/("a"*120),64)[0] 1 1 20 0x0 queue 3
+expect 0 0 eth/fragment6(ipv6/IPv6ExtHdrFragment()/udp/("a"*120),64)[0] 1 1 20 0x0 ptype 0x3e1
+expect 0 0 eth/fragment6(ipv6/IPv6ExtHdrFragment()/udp/("a"*120),64)[0] 1 1 20 0x0 ol_flags 0x80
+expect 0 0 eth/fragment6(ipv6/IPv6ExtHdrFragment()/udp/("a"*120),64)[0] 1 1 20 0x0 rss 0x99f99ccf
+expect 0 0 eth/fragment6(ipv6/IPv6ExtHdrFragment()/udp/("a"*120),64)[1] 1 1 20 0x0 queue 3
+expect 0 0 eth/fragment6(ipv6/IPv6ExtHdrFragment()/udp/("a"*120),64)[1] 1 1 20 0x0 ptype 0x3e1
+expect 0 0 eth/fragment6(ipv6/IPv6ExtHdrFragment()/udp/("a"*120),64)[1] 1 1 20 0x0 ol_flags 0x80
+expect 0 0 eth/fragment6(ipv6/IPv6ExtHdrFragment()/udp/("a"*120),64)[1] 1 1 20 0x0 rss 0x99f99ccf
+expect 0 0 eth 1 1 20 0x0 rss 0
+expect 0 0 eth 1 1 20 0x0 ptype 0x1
+expect 0 0 eth 1 1 20 0x0 ol_flags 0x0
+
+
+
+
+
+expect 0 0 ethb/ip/udp 1 1 20 0x0 rss 0xf4ff21c5
+expect 0 0 ethm/ip/udp 1 1 20 0x0 rss 0xf4ff21c5
+
+
+
+
+
+
+
+
+
+
+
+
+port config all rss none
+stop
+port stop 0
+port start 0
+start
+expect 0 0 eth/ip/udp 1 1 20 0x0 rss 0
+expect 0 0 eth/ip/udp 1 1 20 0x0 ol_flags 0x180
+expect 0 0 eth/ip/udp 1 1 20 0x0 ptype 0x291
+expect 0 0 eth/ip/tcp 1 1 20 0x0 rss 0
+expect 0 0 eth/ip 1 1 20 0x0 rss 0
+expect 0 0 eth/ipv6/udp 1 1 20 0x0 rss 0
+expect 0 0 eth/ipv6/tcp 1 1 20 0x0 rss 0
+expect 0 0 eth/ipv6 1 1 20 0x0 rss 0
+
-- 
2.13.3

[dpdk-dev] [RFC v1 9/9] doc/scapy: add scapy how-to guide

[Xueming Li]

Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 doc/guides/howto/scapy.rst | 300 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 300 insertions(+)
 create mode 100644 doc/guides/howto/scapy.rst

diff --git a/doc/guides/howto/scapy.rst b/doc/guides/howto/scapy.rst
new file mode 100644
index 000000000..498f7918e
--- /dev/null
+++ b/doc/guides/howto/scapy.rst
@@ -0,0 +1,300 @@
+Scapy DPDK extension
+====================
+
+Scapy DPDK extension is a testpmd forwarding engine and a set of command: tx, rx 
+and expect(tx,rx,compare) packet, useful for development, unit test and fast 
+regression.
+
+
+This document introduces basic syntax with examples.
+
+* Scapy usage: http://scapy.readthedocs.io/en/latest/usage.html
+
+Enable python/scapy:
+--------------------------------
+- Remember to install python-libs and scapy using `yum` or `apt`
+- Enable python lib in dpdk configuration, build:
+
+  .. code-block:: c
+ 
+    CONFIG_RTE_LIBRTE_PYTHON=y
+    CONFIG_RTE_LIBRTE_PYTHON_VERSION=python2.7
+
+- Start testpmd in interactive mode
+
+TX - send packet out
+---------------------
+tx <port> <scapy>
+~~~~~~~~~~~~~~~~~
+.. code-block:: c
+
+  # Continuously send simple packet
+  testpmd> tx 0 1
+
+  # Send scapy syntax packet
+  testpmd> tx 0 Ether()/IP()/UDP()/"hello"
+
+  # Flush a subnet
+  testpmd> tx 0 Ether()/IP(dst="192.168.0.1/24")
+  8776640/0 packets sent in 2167534.739us 4.049135mpps
+
+tx <port> <scapy> <count> <verbose>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: text
+
+  # only send 1
+  testpmd> tx 0 Ether()/IP()/"hello" 1 0
+  1/1 packets sent in 1091.912us 0.000916mpps
+
+  # with simple scapy brief
+  testpmd> tx 0 Ether()/IP()/"hello" 1 0x01
+  Ether / 127.0.0.1 > 127.0.0.1 ip / Raw / Padding
+  1/1 packets sent in 1084.420us 0.000922mpps
+
+  # enum
+  testpmd> tx 0 Ether()/IP()/UDP(sport=[54321,54323]) 2 1
+  Ether / IP / UDP 127.0.0.1:54321 > 127.0.0.1:domain / Padding
+  Ether / IP / UDP 127.0.0.1:54323 > 127.0.0.1:domain / Padding
+
+  # range
+  testpmd> tx 0 Ether()/IP()/UDP(sport=(54321,54323)) 3 1
+  Ether / IP / UDP 127.0.0.1:54321 > 127.0.0.1:domain / Padding
+  Ether / IP / UDP 127.0.0.1:54322 > 127.0.0.1:domain / Padding
+  Ether / IP / UDP 127.0.0.1:54323 > 127.0.0.1:domain / Padding
+
+  # generate flows
+  testpmd> tx 0 Ether()/IP(dst="10.0.0.1/31")/TCP(dport=(55555,55556)) 4 2
+  <Ether  dst=00:00:5e:00:01:19 src=80:18:44:e2:6e:fc type=IPv4 |<IP  ihl=5L len=40 frag=0 proto=tcp chksum=0x990f src=10.12.205.180 dst=10.0.0.0 |<TCP  dport=55555 dataofs=5L chksum=0xd50a |<Padding  load='\x00\x00\x00\x00\x00\x00' |>>>>
+  <Ether  dst=00:00:5e:00:01:19 src=80:18:44:e2:6e:fc type=IPv4 |<IP  ihl=5L len=40 frag=0 proto=tcp chksum=0x990f src=10.12.205.180 dst=10.0.0.0 |<TCP  dport=55556 dataofs=5L chksum=0xd509 |<Padding  load='\x00\x00\x00\x00\x00\x00' |>>>>
+  <Ether  dst=00:00:5e:00:01:19 src=80:18:44:e2:6e:fc type=IPv4 |<IP  ihl=5L len=40 frag=0 proto=tcp chksum=0x990e src=10.12.205.180 dst=10.0.0.1 |<TCP  dport=55555 dataofs=5L chksum=0xd509 |<Padding  load='\x00\x00\x00\x00\x00\x00' |>>>>
+  <Ether  dst=00:00:5e:00:01:19 src=80:18:44:e2:6e:fc type=IPv4 |<IP  ihl=5L len=40 frag=0 proto=tcp chksum=0x990e src=10.12.205.180 dst=10.0.0.1 |<TCP  dport=55556 dataofs=5L chksum=0xd508 |<Padding  load='\x00\x00\x00\x00\x00\x00' |>>>>
+  4/4 packets sent in 2923.823us 0.001368mpps
+  
+RX - receive packet
+-------------------
+rx <port>
+~~~~~~~~~
+
+.. code:: text
+
+  # continuously receive on port 0 until Ctrl+C
+  testpmd> rx 0
+  6721984/0 packets received in 1163983.125us 5.774984mpps
+
+rx <port> <count> <timeout(s)> <verbose>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: text
+
+  # Receive 1 packet from port 0 with simple dump
+  testpmd> rx 0 1 0 1
+  Ether / IP / UDP 127.0.0.1:domain > 127.0.0.1:domain / Padding
+  1/1 packets received in 2064638.250us 0.000000mpps
+
+  # receive 1 packet with mbuf header, brief and hex dump
+  testpmd> rx 0 1 10 0x32
+  RX P:0 Q:0 len:60 ptype:0x291 ol_flags:0x180 rss:0x00000000 fdir:0x0
+    ptype: L2_ETHER L3_IPV4_EXT_UNKNOWN L4_UDP
+    ol_flags: PKT_RX_L4_CKSUM_GOOD PKT_RX_IP_CKSUM_GOOD
+  <Ether  dst=ff:ff:ff:ff:ff:ff src=00:00:00:00:00:00 type=IPv4 |<IP  ihl=5L len=28 frag=0 proto=udp chksum=0x7cce src=127.0.0.1 dst=127.0.0.1 |<UDP  len=8 chksum=0x172 |<Padding  load='\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' |>>>>
+  0000   FF FF FF FF FF FF 00 00  00 00 00 00 08 00 45 00   ..............E.
+  0010   00 1C 00 01 00 00 40 11  7C CE 7F 00 00 01 7F 00   ......@.|.......
+  0020   00 01 00 35 00 35 00 08  01 72 00 00 00 00 00 00   ...5.5...r......
+  0030   00 00 00 00 00 00 00 00  00 00 00 00               ............
+  1/1 packets received in 5180381.000us 0.000000mpps
+
+Expect - send, receive and compare
+-------------------------------------
+Need one of following topo to get packet back:
+
+  - VF to VF
+  - use testpmd io forwarding on remote server of a back-to-back connection
+  - set NIC phy in loopback mode
+  - use loopback connector on NIC port
+  - physical connect two port that DPDK support
+
+expect <tx_port> <rx_port> <scapy>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Default timeout: 10ms
+
+.. code:: text
+
+  # basic send
+  testpmd> expect 0 0 Ether()  
+  tx: 1/1 1.211us 0.825826mpps    rx: 1/1 7.571us 0.132077mpps    round: 1/1 7.571us      total: 7.571us 0.132077mpps
+
+.. code:: text
+
+  # if not recevied:
+  testpmd> expect 0 0 Ether()
+  Failed tx: 1/1 8.439us 0.118503mpps     rx: 0/1 10006.879us 0.000000mpps        round: 1/1 10006.879us  total: 10006.879us 0.000000mpps
+
+.. code:: text
+
+  # if packet corrupted, auto diff hex:
+  testpmd> expect 0 0 Ether()/IP()
+  Failed: packet not same:
+  0000        FF FF FF FF FF FF 00 00  00 00 00 00 08 00 45 00   ..............E.
+       0000   00 00 00 00 00 00 FF FF  FF FF FF FF 08 00 45 00   ..............E.
+  0010 0010   00 14 00 01 00 00 40 00  7C E7 7F 00 00 01 7F 00   ......@.|.......
+  0020 0020   00 01 00 00 00 00 00 00  00 00 00 00 00 00 00 00   ................
+  0030 0030   00 00 00 00 00 00 00 00  00 00 00 00               ............
+  RX P:0 Q:0 len:60 ptype:0x691 ol_flags:0x80 rss:0x00000000 fdir:0x0
+    ptype: L2_ETHER L3_IPV4_EXT_UNKNOWN L4_NONFRAG
+    ol_flags: PKT_RX_L4_CKSUM_UNKNOWN PKT_RX_IP_CKSUM_GOOD
+  tx: 1/1 1.494us 0.669507mpps    rx: 1/1 6800.465us 0.000147mpps round: 1/1 6800.465us   total: 6800.465us 0.000147mpps
+
+expect <tx_port> <rx_port> <scapy> <count> <round> <timeout(ms)> <verbose> <field> <val>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: text
+
+  # dump rx header info and assert hw offload flag filed
+  testpmd> expect 0 0 Ether()/IP()/UDP() 1 1 1 0x10  ol_flags 0x182
+  RX P:0 Q:1 len:60 ptype:0x291 ol_flags:0x182 rss:0xf2279e9d fdir:0x0 ptype: L2_ETHER L3_IPV4_EXT_UNKNOWN L4_UDP
+  ol_flags: PKT_RX_RSS_HASH PKT_RX_L4_CKSUM_GOOD PKT_RX_IP_CKSUM_GOOD
+  tx: 1/1 6.240us 0.160256mpps    rx: 1/1 500.597us 0.001998mpps  round: 1/1 500.597us    total: 500.597us 0.001998mpps
+
+Supported field check:
+
+  queue, ptype, rss, fdir, ol_flags, non|0
+
+.. code:: text
+
+  # no packet expect to be received:
+  testpmd> expect 0 0 Ether()/IP()/UDP() 0 1 100 0x10  0 0
+  tx: 1/1 9.058us 0.110397mpps    rx: 0/0 100041.414us 0.000000mpps       round: 1/1 100041.414us total: 100041.414us 0.000000mpps
+
+  # Fail and auto dump if we do receive one:
+  testpmd> expect 0 0 Ether()/IP()/UDP() 0 1 100 0x0  ol_flags 0x182
+  RX P:0 Q:1 len:60 ptype:0x291 ol_flags:0x182 rss:0xf2279e9d fdir:0x0
+  ptype: L2_ETHER L3_IPV4_EXT_UNKNOWN L4_UDP
+  ol_flags: PKT_RX_RSS_HASH PKT_RX_L4_CKSUM_GOOD PKT_RX_IP_CKSUM_GOOD
+  Failed tx: 1/1 5.353us 0.186805mpps     rx: 1/0 100036.852us 0.000010mpps       round: 1/1 100036.852us total: 100036.852us 0.000010mpps
+
+.. code:: bash
+
+  # latency test, tx/rx 10000 rounds
+  testpmd> expect 0 0 Ether() 1 10000 1000 0 0 0
+  tx: 10000/10000 50401.309us 0.198408mpps        rx: 10000/10000 50406.070us 0.198389mpps        round: 10000/10000 5.041us      total: 50406.070us 0.198389mpps
+
+<timeout>:
+
+- 0: endless loop, could be canceled by Ctrl+C
+- integer: seconds(rx) or msecs(expect)
+
+<verbose>: same to global verbose definition
+
+
+Verbose level - global output control
+----------------------------------------------
+
+- set verbose <level>
+
+.. code:: text
+
+   xxxx xxxx xxxx xxxx
+   = == ====  === ====
+   | ||  |    |||  L-- RX 0:mute 1:short 2:brief 3:detail
+   | ||  |    ||L----- RX header dump
+   | ||  |    |L------ RX hex dump
+   | ||  |    L------- Mute succeed expect command, for batch running
+   | ||  L------------ TX 0:mute 1:short 2:brief 3:detail
+   | |L--------------- TX header dump
+   | L---------------- TX hex dump
+   L------------------ Echo CLI to screen during "load" command
+
+py - call python
+---------------- 
+py <commands>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Any python grammar allowed:
+
+.. code:: text
+
+  testpmd> py 1+1
+  2
+
+  testpmd> py hex(12345)
+  '0x3039'
+
+  testpmd> py 0x12345
+  74565
+
+  testpmd> py a=Ether();b=UDP();a/IP()/b; a/IPv6()/b
+  <Ether  type=IPv4 |<IP  frag=0 proto=udp |<UDP  |>>>
+  <Ether  type=IPv6 |<IPv6  nh=UDP |<UDP  |>>>
+
+py shell - enter python shell
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. code:: text
+
+  testpmd> py shell
+  >>> Help(Ether)
+  # "ctrl + d" to quit
+
+py <debug|nodebug>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Toggle python lib debug
+
+PktGen Engine
+--------------------
+A new engine to tx, rx and compare packets based on templates.
+
+pktgen idle <mode>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Behavior of idle:
+
+- 0 - drop: rx only
+- 1 - loopback: rx and send back
+- 2 - forward: using testpmd port-queue mapping
+- 3 - switch: switch mac address and send back
+
+Batch Test
+------------------------
+
+load <file>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+load and run testpmd CLI batch in mute
+
+.. code:: text
+
+  testpmd> load test/expect/init.exp
+  Change verbose level from 0 to 64
+  Read CLI commands from test/expect/init.exp
+
+  # verify 
+  testpmd>py eth
+  <Ether  dst=aa:bb:cc:dd:ee:ff src=00:11:22:33:44:55 |>
+
+set verbose 0x8000
+~~~~~~~~~~~~~~~~~~~
+Set testpmd batch file loading with CLI echo to screen, easy to find source CLI if any error occurs.
+
+.. code:: text
+
+  testpmd> set verbose 0x8000
+  testpmd> load test/expect/rx.exp
+  testpmd> py eth = Ether(src="00:11:22:33:44:55",dst="aa:bb:cc:dd:ee:ff")
+  testpmd> py ethb = Ether(src="00:11:22:33:44:55",dst="ff:ff:ff:ff:ff:ff")
+  ...
+
+Known issues/TODO:
+---------------------
+- Code format
+- TX offload
+- Jumbo packet send
+- LRO rx
+- Multi-queue for max throughput
+- Dynamic packet template - slow but flexible
+- Test suit with summary
+- mbuf packet type in scapy?
+- dpdk wrapper for python - due to complexity to expand testpmd CLI, how about manipulating DPDK in python unit test framework?
+
+Design consideration:
+---------------------------
+- Syntax flexibility from Scapy
+- Speed of DPDK
+- Quick batch regression for developer to avoid anything broken
-- 
2.13.3

[dpdk-dev] [RFC PATCH 2/2] app/testpmd: add scapy command as pkt template

[Xueming Li]

Invoke scapy to generate packet templates for each queue of specific
port, command format:
    scapy <port> <pattern>
Example:
    scapy 0 Ether()/IP()/GRE()/IP()/UDP(dport=(2,5))/"cool"

Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 app/test-pmd/Makefile  |  5 +++
 app/test-pmd/cmdline.c | 99 +++++++++++++++++++++++++++++++++++++++++++++++++-
 config/common_base     |  2 +
 3 files changed, 105 insertions(+), 1 deletion(-)

diff --git a/app/test-pmd/Makefile b/app/test-pmd/Makefile
index 2c50f68a9..220be17f2 100644
--- a/app/test-pmd/Makefile
+++ b/app/test-pmd/Makefile
@@ -91,6 +91,11 @@ endif
 
 endif
 
+ifeq ($(CONFIG_RTE_TEST_PMD_SCAPY),y)
+LDLIBS += -l$(CONFIG_RTE_PYTHON)
+CFLAGS += -I/usr/include/$(CONFIG_RTE_PYTHON)
+endif
+
 CFLAGS_cmdline.o := -D_GNU_SOURCE
 
 include $(RTE_SDK)/mk/rte.app.mk
diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index bb01e989a..b6df7ea75 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -33,6 +33,9 @@
  */
 
 #include <stdarg.h>
+#ifdef RTE_TEST_PMD_SCAPY
+#include <Python.h>
+#endif
 #include <errno.h>
 #include <stdio.h>
 #include <stdint.h>
@@ -48,7 +51,6 @@
 #endif
 #endif
 #include <netinet/in.h>
-
 #include <sys/queue.h>
 
 #include <rte_common.h>
@@ -15391,6 +15393,98 @@ cmdline_parse_inst_t cmd_load_from_file = {
 	},
 };
 
+#ifdef RTE_TEST_PMD_SCAPY
+/* Common result structure for file commands */
+struct cmd_scapy_result {
+	cmdline_fixed_string_t scapy;
+	portid_t port;
+	cmdline_fixed_string_t pattern;
+};
+
+/* Common CLI fields for file commands */
+cmdline_parse_token_string_t cmd_scapy_cmd =
+	TOKEN_STRING_INITIALIZER(struct cmd_scapy_result, scapy, "scapy");
+cmdline_parse_token_num_t cmd_scapy_port =
+	TOKEN_NUM_INITIALIZER(struct cmd_scapy_result, port, UINT16);
+cmdline_parse_token_string_t cmd_scapy_pattern =
+	TOKEN_STRING_INITIALIZER(struct cmd_scapy_result, pattern, NULL);
+
+static void
+cmd_scapy_parsed(
+	void *parsed_result,
+	__attribute__((unused)) struct cmdline *cl,
+	__attribute__((unused)) void *data)
+{
+	struct cmd_scapy_result *res = parsed_result;
+	PyObject *mod, *gdict, *ldict, *list, *item;
+	int i, socket;
+	int len = 0;
+	struct rte_mbuf *mbuf;
+	char str[sizeof(res->pattern) + 128];
+
+	socket = rte_eth_dev_socket_id(res->port);
+	/* Clean port templates */
+	for (i = 0;  i < RTE_MAX_QUEUES_PER_PORT; i++)
+		if (pkt_templ[res->port][i])
+			rte_mbuf_raw_free(pkt_templ[res->port][i]);
+	/* Invoke scapy */
+	Py_Initialize();
+	if (!Py_IsInitialized())
+		goto err;
+	if (PyRun_SimpleString("import sys;from scapy.all import *"))
+		goto err;
+	mod = PyImport_AddModule("__main__");
+	if (!mod)
+		goto err;
+	gdict = PyModule_GetDict(mod);
+	ldict = PyDict_New();
+	if (!gdict || !ldict)
+		goto err;
+	snprintf(str, sizeof(str), "p = %s;r = [str(x) for x in p];\n",
+		 res->pattern);
+	PyRun_String(str, Py_file_input, gdict, ldict);
+	list = PyDict_GetItem(ldict, PyString_FromString("r"));
+	if (!list)
+		goto err;
+	for (i = 0; i < PyList_Size(list) && i < RTE_MAX_QUEUES_PER_PORT; i++) {
+		item = PyList_GET_ITEM(list, i);
+		/* allocate mbuf & copy raw pkt */
+		mbuf = rte_mbuf_raw_alloc(mbuf_pool_find(socket));
+		if (!mbuf)
+			goto end;
+		rte_memcpy(rte_pktmbuf_mtod_offset(mbuf, void *, 0),
+			   PyString_AsString(item),
+			   PyString_Size(item));
+		len = PyString_Size(item);
+		mbuf->data_len = len > TXONLY_DEF_PACKET_LEN ?
+				 len : TXONLY_DEF_PACKET_LEN;
+		pkt_templ[res->port][i] = mbuf;
+	}
+	if (len > TXONLY_DEF_PACKET_LEN)
+		printf("%d templates saved, size: %d\n", i, len);
+	else
+		printf("%d templates saved, size: %d -> %d\n", i, len,
+		       TXONLY_DEF_PACKET_LEN);
+	goto end;
+err:
+	printf("Error!\n");
+end:
+	Py_Finalize();
+}
+
+cmdline_parse_inst_t cmd_scapy = {
+	.f = cmd_scapy_parsed,
+	.data = NULL,
+	.help_str = "scapy <port> Ether()/IP()/GRE()/IP()/UDP(dport=(2,5))/\"cool\"",
+	.tokens = {
+		(void *)&cmd_scapy_cmd,
+		(void *)&cmd_scapy_port,
+		(void *)&cmd_scapy_pattern,
+		NULL,
+	},
+};
+#endif
+
 /* ******************************************************************************** */
 
 /* list of instructions */
@@ -15399,6 +15493,9 @@ cmdline_parse_ctx_t main_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_help_long,
 	(cmdline_parse_inst_t *)&cmd_quit,
 	(cmdline_parse_inst_t *)&cmd_load_from_file,
+#ifdef RTE_TEST_PMD_SCAPY
+	(cmdline_parse_inst_t *)&cmd_scapy,
+#endif
 	(cmdline_parse_inst_t *)&cmd_showport,
 	(cmdline_parse_inst_t *)&cmd_showqueue,
 	(cmdline_parse_inst_t *)&cmd_showportall,
diff --git a/config/common_base b/config/common_base
index d9471e806..4abb6f2e1 100644
--- a/config/common_base
+++ b/config/common_base
@@ -785,6 +785,8 @@ CONFIG_RTE_APP_TEST_RESOURCE_TAR=n
 CONFIG_RTE_TEST_PMD=y
 CONFIG_RTE_TEST_PMD_RECORD_CORE_CYCLES=n
 CONFIG_RTE_TEST_PMD_RECORD_BURST_STATS=n
+CONFIG_RTE_TEST_PMD_SCAPY=y
+CONFIG_RTE_PYTHON=python2.7
 
 #
 # Compile the crypto performance application
-- 
2.13.3

Re: [dpdk-dev] [RFC PATCH 0/2] using scapy to generate packet templates

[Van Haaren, Harry]

> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Xueming Li
> Sent: Thursday, October 19, 2017 3:07 PM
> To: Wu, Jingjing <jingjing.wu@intel.com>
> Cc: Xueming Li <xuemingl@mellanox.com>; dev@dpdk.org
> Subject: [dpdk-dev] [RFC PATCH 0/2] using scapy to generate packet templates

Hi,

First off - cool to make DPDK more dynamic with Scapy, and I see value in
"somehow" enabling DPDK with Scapy.

> The target is to make testpmd flexible to generate all kinds of packet
> in just one line. Examples:
> 
> # send UDP packet to port 0 queue 0
> testpmd> scapy 0 Ether()/IP()/UDP()/"hello_world"
> 
> # send 4 L3 VXLAN packets with inner UDP dport 1-4 to port 1 queue 1-4, one
> packet per queue
> testpmd> scapy 1 Ether()/IP()/UDP()/VXLAN()/IPv6()/UDP(dport=(1,4))/"cool"
> 
> # Random
> testpmd> scapy 2
> Ether(src=RandMAC())/IP(dst=RandIP())/TCP(sport=RandShort(),dport=0x1234)
> 
> This patch introduce scapy - a python tool to generate static packet
> templates.
> Txonly forwarding mode is modified to support template if available.


Testpmd is a good use-case, but perhaps we're limiting the scope of DPDK/Scapy
if we only integrate with Testpmd. What if I wanted to easily generate some traffic from a sample app?

As a suggestion, what about building a library that provides Scapy functionality,
allowing any application to easily generate mbufs using the rte_scapy* functions:

/* demo usage */
rte_scapy *scapy = rte_scapy_init();
rte_mbuf *m = rte_scapy_mbuf_from_str(scapy, "Ether()/IP()/UDP()");


Testpmd could then use the rte_scapy library, passing in any parameters
after   testpmd> scapy   command as a string?

Had you already considered this library type design?


> Two new configuration:
> CONFIG_RTE_TEST_PMD_SCAPY=y
> CONFIG_RTE_PYTHON=python2.7
> 
> Scapy quick demo: http://www.secdev.org/projects/scapy/demo.html
> 
> There are 2 other potential solution to support more fancy sending in the
> future:
> 1. Invoke scapy and expose mbuf hugepage pointers to python
>    This should allow more flexible and dynamic processing
> 2. Scapy uses wrapped DPDK port as regular port
>    Make scapy capable to send high speed streams.

Not a strong opinion, but a gut feeling;
I see more value in generating packets with Python and using them from C
than giving Python full access to DPDK internals and hugepages.

Regards, -Harry


> Xueming Li (2):
>   app/testpmd: add packet template
>   app/testpmd: add scapy command as pkt template
> 
>  app/test-pmd/Makefile  |  5 +++
>  app/test-pmd/cmdline.c | 99
> +++++++++++++++++++++++++++++++++++++++++++++++++-
>  app/test-pmd/testpmd.c |  3 ++
>  app/test-pmd/testpmd.h |  1 +
>  app/test-pmd/txonly.c  | 42 +++++++++++++--------
>  config/common_base     |  2 +
>  6 files changed, 135 insertions(+), 17 deletions(-)
> 
> --
> 2.13.3

Re: [dpdk-dev] [RFC PATCH 0/2] using scapy to generate packet templates

[Xueming(Steven) Li]

Beside generating packets, there are more things can be done here with scapy/python, might be useful for developing:

1. generate limited packet and send: 
	tx {pattern} {port} {count}
2. parse and dump packet from binary to Scapy format.
	rx {pattern} {port}
3. loopback send and receive, compare with same packet:
	expect {pattern} {port} {queue}
4. send and expect no packet:
	unexpect {pattern} {port} 

When developing tunnel protocol, I found it better to design a bundle of test cases and run it easily anytime, especially after any PMD bug fix, there are tons of case combination to test, a few examples:
1. setup different inner/outer RSS mode, testing different packet type, expecting different queue and rss hash result from mbuf header.
2. verify csum of out and inner packet.
3. verify FDIR of mbuf header

Embedded Python could help here:
1. Using unit test framework to manage, run and report cases
2. export mbuf and limited dpdk api to python, benefit from python syntax sugar:
	dpdk.tx("mbuf(nsegs=3, pktlen=8000)/Ether()/IP(chksum=0xbad)", count=1000, port=1, verbose=1)
	dpdk.expect(pkt1, pkt2, sport=1, dport=2)
	

A big disadvantage is that compiler can't find out disconnection between C api and exposed python api(scripts), have to run unit test to know it.

> -----Original Message-----
> From: Van Haaren, Harry [mailto:harry.van.haaren@intel.com]
> Sent: Thursday, October 19, 2017 11:22 PM
> To: Xueming(Steven) Li <xuemingl@mellanox.com>; Wu, Jingjing
> <jingjing.wu@intel.com>
> Cc: dev@dpdk.org
> Subject: RE: [dpdk-dev] [RFC PATCH 0/2] using scapy to generate packet
> templates
> 
> > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Xueming Li
> > Sent: Thursday, October 19, 2017 3:07 PM
> > To: Wu, Jingjing <jingjing.wu@intel.com>
> > Cc: Xueming Li <xuemingl@mellanox.com>; dev@dpdk.org
> > Subject: [dpdk-dev] [RFC PATCH 0/2] using scapy to generate packet
> > templates
> 
> Hi,
> 
> First off - cool to make DPDK more dynamic with Scapy, and I see value in
> "somehow" enabling DPDK with Scapy.
> 
> > The target is to make testpmd flexible to generate all kinds of packet
> > in just one line. Examples:
> >
> > # send UDP packet to port 0 queue 0
> > testpmd> scapy 0 Ether()/IP()/UDP()/"hello_world"
> >
> > # send 4 L3 VXLAN packets with inner UDP dport 1-4 to port 1 queue
> > 1-4, one packet per queue
> > testpmd> scapy 1
> Ether()/IP()/UDP()/VXLAN()/IPv6()/UDP(dport=(1,4))/"cool"
> >
> > # Random
> > testpmd> scapy 2
> > Ether(src=RandMAC())/IP(dst=RandIP())/TCP(sport=RandShort(),dport=0x12
> > 34)
> >
> > This patch introduce scapy - a python tool to generate static packet
> > templates.
> > Txonly forwarding mode is modified to support template if available.
> 
> 
> Testpmd is a good use-case, but perhaps we're limiting the scope of
> DPDK/Scapy if we only integrate with Testpmd. What if I wanted to easily
> generate some traffic from a sample app?
> 
> As a suggestion, what about building a library that provides Scapy
> functionality, allowing any application to easily generate mbufs using
> the rte_scapy* functions:
> 
> /* demo usage */
> rte_scapy *scapy = rte_scapy_init();
> rte_mbuf *m = rte_scapy_mbuf_from_str(scapy, "Ether()/IP()/UDP()");
> 
> 
> Testpmd could then use the rte_scapy library, passing in any parameters
> after   testpmd> scapy   command as a string?
> 
> Had you already considered this library type design?
> 

Agree, good suggestion.

> 
> > Two new configuration:
> > CONFIG_RTE_TEST_PMD_SCAPY=y
> > CONFIG_RTE_PYTHON=python2.7
> >
> > Scapy quick demo:
> > https://emea01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fwww.
> > secdev.org%2Fprojects%2Fscapy%2Fdemo.html&data=02%7C01%7Cxuemingl%40me
> > llanox.com%7Cd6c3f991ab8247d35fca08d5170530ad%7Ca652971c7d2e4d9ba6a4d1
> > 49256f461b%7C0%7C0%7C636440233429123890&sdata=QLZinBnpnaJDlalLypdj8KW1
> > ZWi0yja3EZvD7iT%2F9AA%3D&reserved=0
> >
> > There are 2 other potential solution to support more fancy sending in
> > the
> > future:
> > 1. Invoke scapy and expose mbuf hugepage pointers to python
> >    This should allow more flexible and dynamic processing 2. Scapy
> > uses wrapped DPDK port as regular port
> >    Make scapy capable to send high speed streams.
> 
> Not a strong opinion, but a gut feeling; I see more value in generating
> packets with Python and using them from C than giving Python full access
> to DPDK internals and hugepages.

Both direction attractive enough, considering unit test, have to exposing warping api, mbuf and ethdev apis more or less...

> 
> Regards, -Harry
> 
> 
> > Xueming Li (2):
> >   app/testpmd: add packet template
> >   app/testpmd: add scapy command as pkt template
> >
> >  app/test-pmd/Makefile  |  5 +++
> >  app/test-pmd/cmdline.c | 99
> > +++++++++++++++++++++++++++++++++++++++++++++++++-
> >  app/test-pmd/testpmd.c |  3 ++
> >  app/test-pmd/testpmd.h |  1 +
> >  app/test-pmd/txonly.c  | 42 +++++++++++++--------
> >  config/common_base     |  2 +
> >  6 files changed, 135 insertions(+), 17 deletions(-)
> >
> > --
> > 2.13.3