From: Hekai Wang <hewang@redhat.com>
To: dev@dpdk.org
Subject: [dpdk-dev] Fwd: SR-IOV dpdk testpmd bonding does not work
Date: Wed, 28 Feb 2018 14:30:38 +0800 [thread overview]
Message-ID: <CAOzSe3h9tZnLwDQ5dHD2BS-tCmWap6aopuzL4n0d=SUcpUtEMg@mail.gmail.com> (raw)
In-Reply-To: <CAOzSe3gmg0sdvtvwefhBLEJrtQwj6vEZWMDr=M+jrGSdOMdoEQ@mail.gmail.com>
---------- Forwarded message ----------
From: Hekai Wang <hewang@redhat.com>
Date: 2018-02-28 14:28 GMT+08:00
Subject: SR-IOV dpdk testpmd bonding does not work
To: test-report@dpdk.org
Hi
I have a problem with dpdk testpmd bonding,I think may be a bug
Test env
under test machine and traffic sender machine connect directly
each machine have two ports
[xxx]# dpdk-devbind -s
Network devices using DPDK-compatible driver
============================================
0000:05:10.0 '82599 Ethernet Controller Virtual Function' drv=vfio-pci
unused=ixgbevf
0000:05:10.1 '82599 Ethernet Controller Virtual Function' drv=vfio-pci
unused=ixgbevf
Network devices using kernel driver
===================================
0000:01:00.0 'NetXtreme BCM5720 Gigabit Ethernet PCIe' if=em1 drv=tg3
unused=vfio-pci *Active*
0000:01:00.1 'NetXtreme BCM5720 Gigabit Ethernet PCIe' if=em2 drv=tg3
unused=vfio-pci
0000:02:00.0 'NetXtreme BCM5720 Gigabit Ethernet PCIe' if=em3 drv=tg3
unused=vfio-pci
0000:02:00.1 'NetXtreme BCM5720 Gigabit Ethernet PCIe' if=em4 drv=tg3
unused=vfio-pci
0000:05:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' if=p7p1
drv=ixgbe unused=vfio-pci
0000:05:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' if=p7p2
drv=ixgbe unused=vfio-pci
Other network devices
=====================
<none>
Crypto devices using DPDK-compatible driver
===========================================
<none>
Crypto devices using kernel driver
==================================
<none>
Other crypto devices
====================
<none>
[root@dell-per730-18 ~]# ethtool -i p7p1
driver: ixgbe
version: 4.4.0-k-rh7.4
firmware-version: 0x61ae0001
expansion-rom-version:
bus-info: 0000:05:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: no
[root@dell-per730-18 ~]# ethtool -i p7p2
driver: ixgbe
version: 4.4.0-k-rh7.4
firmware-version: 0x61ae0001
expansion-rom-version:
bus-info: 0000:05:00.1
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: no
**************************
ip li info
6: p7p1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode
DEFAULT qlen 1000
link/ether 90:e2:ba:29:bf:14 brd ff:ff:ff:ff:ff:ff
vf 0 MAC 2e:c7:ba:0c:73:f8, spoof checking off, link-state auto, trust
on, query_rss off
7: p7p2: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc mq state
UP mode DEFAULT qlen 1000
link/ether 90:e2:ba:29:bf:15 brd ff:ff:ff:ff:ff:ff
vf 0 MAC c6:24:28:7a:c8:ec, spoof checking off, link-state auto, trust
on, query_rss off
So , current p7p1 and p7p2 with one vf with each port.
1. bond each vf to vfio-pci driver
[xxx]# dpdk-devbind -s
Network devices using DPDK-compatible driver
============================================
0000:05:10.0 '82599 Ethernet Controller Virtual Function' drv=vfio-pci
unused=ixgbevf
0000:05:10.1 '82599 Ethernet Controller Virtual Function' drv=vfio-pci
unused=ixgbevf
2. create a VM
#need qemu-img create a image first
bin/bash -c "sudo -E taskset -c 4,6,30 /usr/libexec/qemu-kvm -m 4096 -smp 3
-cpu host,migratable=off -drive if=ide,file=rhel1Q.qcow2 -boot c
--enable-kvm -monitor unix:/tmp/vm0monitor,server,nowait -object
memory-backend-file,id=mem,size=4096M,mem-path=/dev/hugepages,share=on
-numa node,memdev=mem -mem-prealloc -nographic -vnc :0 -name Client0
-snapshot -net none -no-reboot -device vfio-pci,host=0000:05:10.0 -device
vfio-pci,host=0000:05:10.1"
Guest(VM) Command
iptables -t filter -P INPUT ACCEPT
iptables -t filter -P FORWARD ACCEPT
iptables -t filter -P OUTPUT ACCEPT
iptables -t mangle -P PREROUTING ACCEPT
iptables -t mangle -P INPUT ACCEPT
iptables -t mangle -P FORWARD ACCEPT
iptables -t mangle -P OUTPUT ACCEPT
iptables -t mangle -P POSTROUTING ACCEPT
iptables -t nat -P PREROUTING ACCEPT
iptables -t nat -P INPUT ACCEPT
iptables -t nat -P OUTPUT ACCEPT
iptables -t nat -P POSTROUTING ACCEPT
iptables -t filter -F
iptables -t filter -X
iptables -t mangle -F
iptables -t mangle -X
iptables -t nat -F
iptables -t nat -X
ip6tables -t filter -P INPUT ACCEPT
ip6tables -t filter -P FORWARD ACCEPT
ip6tables -t filter -P OUTPUT ACCEPT
ip6tables -t mangle -P PREROUTING ACCEPT
ip6tables -t mangle -P INPUT ACCEPT
ip6tables -t mangle -P FORWARD ACCEPT
ip6tables -t mangle -P OUTPUT ACCEPT
ip6tables -t mangle -P POSTROUTING ACCEPT
ip6tables -t nat -P PREROUTING ACCEPT
ip6tables -t nat -P INPUT ACCEPT
ip6tables -t nat -P OUTPUT ACCEPT
ip6tables -t nat -P POSTROUTING ACCEPT
ip6tables -t filter -F
ip6tables -t filter -X
ip6tables -t mangle -F
ip6tables -t mangle -X
ip6tables -t nat -F
ip6tables -t nat -X
sysctl vm.nr_hugepages=1
mkdir -p /dev/hugepages
mount -t hugetlbfs hugetlbfs /dev/hugepages
cat /proc/meminfo
# at this time , download dpdk and compile and install it , I use dpdk17.11
ip link show
ip ad show
cat /proc/cmdline
dpdk-devbind --status
ifdown eth0
ifdown eth1
dpdk-bind --status
dpdk-devbind -u 00:03.0 00:04.0
modprobe -r vfio
modprobe -r vfio_iommu_type1
modprobe vfio enable_unsafe_noiommu_mode=Y
modprobe vfio-pci
dpdk-devbind -b vfio-pci 00:03.0 00:04.0
dpdk-devbind --status
testpmd -l 0,1,2 -n 4 --socket-mem 512 -- --burst=64 -i --txqflags=0xf00
--disable-hw-vlan --nb-cores=2, --txq=1 --rxq=1 --rxd=512 --txd=512
--port-topology=chained
set fwd macswap
create bonded device 1 0
set bonding mode 1 2
set bonding mon_period 2 100
add bonding slave 0 2
add bonding slave 1 2
set bonding primary 0 2
port start 2
set fwd macswap
start
show port info all
testpmd> show port info all
********************* Infos for port 0 *********************
MAC address: 2E:C7:BA:0C:73:F8
Driver name: net_ixgbe_vf
Connect to socket: 0
memory allocation on the socket: 0
Link status: up
Link speed: 10000 Mbps
Link duplex: full-duplex
MTU: 1500
Promiscuous mode: disabled
Allmulticast mode: disabled
Maximum number of MAC addresses: 128
Maximum number of MAC addresses of hash filtering: 4096
VLAN offload:
strip off
filter off
qinq(extend) off
No flow type is supported.
Max possible RX queues: 4
Max possible number of RXDs per queue: 4096
Min possible number of RXDs per queue: 32
RXDs number alignment: 8
Max possible TX queues: 4
Max possible number of TXDs per queue: 4096
Min possible number of TXDs per queue: 32
TXDs number alignment: 8
********************* Infos for port 1 *********************
MAC address: C6:24:28:7A:C8:EC
Driver name: net_ixgbe_vf
Connect to socket: 0
memory allocation on the socket: 0
Link status: up
Link speed: 10000 Mbps
Link duplex: full-duplex
MTU: 1500
Promiscuous mode: disabled
Allmulticast mode: disabled
Maximum number of MAC addresses: 128
Maximum number of MAC addresses of hash filtering: 4096
VLAN offload:
strip off
filter off
qinq(extend) off
No flow type is supported.
Max possible RX queues: 4
Max possible number of RXDs per queue: 4096
Min possible number of RXDs per queue: 32
RXDs number alignment: 8
Max possible TX queues: 4
Max possible number of TXDs per queue: 4096
Min possible number of TXDs per queue: 32
TXDs number alignment: 8
********************* Infos for port 2 *********************
MAC address: 2E:C7:BA:0C:73:F8
Driver name: net_bonding
Connect to socket: 0
memory allocation on the socket: 0
Link status: up
Link speed: 10000 Mbps
Link duplex: full-duplex
MTU: 1500
Promiscuous mode: enabled
Allmulticast mode: disabled
Maximum number of MAC addresses: 1
Maximum number of MAC addresses of hash filtering: 0
VLAN offload:
strip off
filter off
qinq(extend) off
No flow type is supported.
Max possible RX queues: 4
Max possible number of RXDs per queue: 65535
Min possible number of RXDs per queue: 0
RXDs number alignment: 1
Max possible TX queues: 4
Max possible number of TXDs per queue: 65535
Min possible number of TXDs per queue: 0
TXDs number alignment: 1
***************************************************************
another side send traffic
18: p7p1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq portid
6805ca37da40 state UP mode DEFAULT qlen 1000
link/ether 68:05:ca:37:da:40 brd ff:ff:ff:ff:ff:ff
19: p7p2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq portid
6805ca37da41 state UP mode DEFAULT qlen 1000
link/ether 68:05:ca:37:da:41 brd ff:ff:ff:ff:ff:ff
[root@dell-per730-21 ~]# scapy
INFO: Can't import python gnuplot wrapper . Won't be able to plot.
INFO: Can't import PyX. Won't be able to use psdump() or pdfdump().
INFO: Can't import python Crypto lib. Won't be able to decrypt WEP.
INFO: Can't import python Crypto lib. Disabled certificate manipulation
tools
Welcome to Scapy (2.2.0)
>>> kaizi = Ether()
>>> kaizi.dst="2E:C7:BA:0C:73:F8"
>>> kaizi.src="68:05:ca:37:da:40"
>>> kaizi.show()
###[ Ethernet ]###
dst= 2E:C7:BA:0C:73:F8
src= 68:05:ca:37:da:40
type= 0x0
>>> sendp(kaizi,loop=100,inter=1,iface="p7p1")
..................................^C
Sent 34 packets.
at this point use tcpdump catch the packets find that only have send
packest and no response package ......
[root@dell-per730-21 ~]# tcpdump -nei p7p1
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on p7p1, link-type EN10MB (Ethernet), capture size 262144 bytes
01:19:31.535176 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:32.536383 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:33.537510 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:34.538711 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:35.539893 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:36.541095 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:37.542220 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:38.543569 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:39.544805 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:40.546020 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:41.547158 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:42.548367 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:43.549577 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:44.550787 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:45.551921 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:46.553129 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:47.554335 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:48.555541 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:49.556674 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:50.557878 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:51.559082 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:52.560312 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:53.561446 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
01:19:54.562657 68:05:ca:37:da:40 > 2e:c7:ba:0c:73:f8, 802.3, length 0:
[|llc]
at the same time ,use tcpdump capture the packets with another port p7p2
,find that the response packets at this port , so strange ...
because the under test machine vm's testpmd bond mode is active backup mode
.I do not think this port should receive packets .but it is true.
[root@dell-per730-21 ~]# tcpdump -nei p7p2
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on p7p2, link-type EN10MB (Ethernet), capture size 262144 bytes
01:19:50.557897 2e:c7:ba:0c:73:f8 > 68:05:ca:37:da:40, 802.3, length 0:
LLC, dsap Null (0x00) Individual, ssap Null (0x00) Command, ctrl 0x0000:
Information, send seq 0, rcv seq 0, Flags [Command], length 46
0x0000: 0000 0000 0000 0000 0000 0000 0000 0000 ................
0x0010: 0000 0000 0000 0000 0000 0000 0000 0000 ................
0x0020: 0000 0000 0000 0000 0000 0000 0000 ..............
01:19:51.559097 2e:c7:ba:0c:73:f8 > 68:05:ca:37:da:40, 802.3, length 0:
LLC, dsap Null (0x00) Individual, ssap Null (0x00) Command, ctrl 0x0000:
Information, send seq 0, rcv seq 0, Flags [Command], length 46
0x0000: 0000 0000 0000 0000 0000 0000 0000 0000 ................
0x0010: 0000 0000 0000 0000 0000 0000 0000 0000 ................
0x0020: 0000 0000 0000 0000 0000 0000 0000 ..............
01:19:52.560329 2e:c7:ba:0c:73:f8 > 68:05:ca:37:da:40, 802.3, length 0:
LLC, dsap Null (0x00) Individual, ssap Null (0x00) Command, ctrl 0x0000:
Information, send seq 0, rcv seq 0, Flags [Command], length 46
0x0000: 0000 0000 0000 0000 0000 0000 0000 0000 ................
0x0010: 0000 0000 0000 0000 0000 0000 0000 0000 ................
0x0020: 0000 0000 0000 0000 0000 0000 0000 ..............
at the same time , we watch the under test machine use testpmd command
************************************************************
************************************************************
****************************************************************
************************************************************
************************************************************
****************************************************************
testpmd> show port stats all
######################## NIC statistics for port 0
########################
RX-packets: 34 RX-missed: 0 RX-bytes: 2040
RX-errors: 0
RX-nombuf: 0
TX-packets: 0 TX-errors: 0 TX-bytes: 0
Throughput (since last show)
Rx-pps: 0
Tx-pps: 0
############################################################
################
######################## NIC statistics for port 1
########################
RX-packets: 0 RX-missed: 0 RX-bytes: 0
RX-errors: 0
RX-nombuf: 0
TX-packets: 34 TX-errors: 0 TX-bytes: 2040
Throughput (since last show)
Rx-pps: 0
Tx-pps: 0
############################################################
################
######################## NIC statistics for port 2
########################
RX-packets: 34 RX-missed: 0 RX-bytes: 2040
RX-errors: 0
RX-nombuf: 0
TX-packets: 34 TX-errors: 0 TX-bytes: 2040
Throughput (since last show)
Rx-pps: 0
Tx-pps: 0
############################################################
################
follow the stat result , I find that port 0 only receive packets and port 1
only send packets. but current the bond mode is below (active-backup mode )
testpmd> show bonding config 2
Bonding mode: 1
Slaves (2): [0 1]
Active Slaves (2): [0 1]
Primary: [0]
I do not know why the result is this .could some one can help me . Thanks.
2E:C7:BA:0C:73:F8
next parent reply other threads:[~2018-02-28 6:30 UTC|newest]
Thread overview: 3+ messages / expand[flat|nested] mbox.gz Atom feed top
[not found] <CAOzSe3gmg0sdvtvwefhBLEJrtQwj6vEZWMDr=M+jrGSdOMdoEQ@mail.gmail.com>
2018-02-28 6:30 ` Hekai Wang [this message]
2018-03-02 18:02 ` Ferruh Yigit
2018-03-03 12:05 ` Hekai Wang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to='CAOzSe3h9tZnLwDQ5dHD2BS-tCmWap6aopuzL4n0d=SUcpUtEMg@mail.gmail.com' \
--to=hewang@redhat.com \
--cc=dev@dpdk.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).