Re: [dpdk-dev] [dpdk-web] [PATCH v1] add dpdk-quickstart python script

["Hunt, David" <david.hunt@intel.com>]

Hi,

I'm just wondering if there is anything I can do to help move this 
website patch forward? Any suggestions?

Regards,
Dave.



On 3/8/2018 12:41 PM, david.hunt at intel.com (David Hunt) wrote:
> This patch contains two section.
>    1. Updates to the existing quick-start.html page giving
>       infomration on the new dpdk-quickstart.py script.
>    2. The dpdk-quickstart.py script itself.
>
> 1. The Quick start section contains some instructions for
> building DPDK and running TestPMD.
> While this is still useful, automating these steps
> through a script would provide a much faster and painless
> way to have DPDK up and running. The new dpdk-quickstart.py
> aims to address this.
>
> 2. This script performs the following:
> - Gets the latest DPDK release
> - Gets the necessary dependencies (if needed)
> - Builds DPDK with the default target
> - Sets up hugepages
> - Helps the user to select the ports to use on DPDK
> - Runs TestPMD, showing packet forwarding between two ports
>
> Signed-off-by: Pablo de Lara <pablo.de.lara.guarch at intel.com>
> Signed-off-by: Kirill Rybalchenko <kirill.rybalchenko at intel.com>
> Signed-off-by: David Hunt <david.hunt at intel.com>
> ---
>   doc/quick-start.html       |  25 +
>   scripts/dpdk-quickstart.py | 996 +++++++++++++++++++++++++++++++++++++
>   2 files changed, 1021 insertions(+)
>   create mode 100755 scripts/dpdk-quickstart.py
>
> diff --git a/doc/quick-start.html b/doc/quick-start.html
> index 85551fa..fa49932 100644
> --- a/doc/quick-start.html
> +++ b/doc/quick-start.html
> @@ -39,6 +39,31 @@
>   		</header>
>   		<section>
>   			<h2>Quick start</h2>
> +			<p><em>The dpdk-quickstart script is a python script which
> +			provides a quick way to get DPDK up and running.</em></p>
> +
> +			<p>The script can be downloaded here: <a href="/scripts/dpdk-quickstart.py">dpdk-quickstart.py</a>
> +
> +			<p>The script performs the following operations, on a guided and
> +			interactive graphical interface:</p>
> +
> +			<ul>
> +			  <li>Gets the latest DPDK release from the main git repository</li>
> +			  <li>Checks and downloads the necessary dependencies</li>
> +			  <li>Builds DPDK with the default target</li>
> +			  <li>Sets up the minimum amount of hugepages to run TestPMD (a basic
> +			L2 forwarding application)</li>
> +			  <li>Helps the user to select the ports to use on DPDK (including
> +			loopback detection between ports)</li>
> +			  <li>Runs TestPMD, to show packet forwarding between two ports
> +			(virtual interfaces can also be used, if no ports connected with a
> +			loopback are available).</li>
> +			</ul>
> +			<p>Note that this script is only supported on Fedora and Ubuntu.<span style="font-style: italic;"></span></p>
> +			<p><span style="font-style: italic;"><br>
> +			</span></p>
> +			<p><span style="font-style: italic;"></span>Alternatively, DPDK can be
> +			installed and configured manually, with the following instructions:</p>
>   			<p><em>"A simple forwarding test with pcap PMD which works with any NIC (with performance penalties)"</em></p>
>   			<p>Extract sources.</p>
>   			<pre>
> diff --git a/scripts/dpdk-quickstart.py b/scripts/dpdk-quickstart.py
> new file mode 100755
> index 0000000..5fe82d0
> --- /dev/null
> +++ b/scripts/dpdk-quickstart.py
> @@ -0,0 +1,996 @@
> +#!/bin/sh
> +''''which python3 >/dev/null 2>&1 && exec python3 "$0" "$@" # '''
> +''''which python2 >/dev/null 2>&1 && exec python2 "$0" "$@" # '''
> +
> +''''which python  >/dev/null 2>&1 && exec python  "$0" "$@" # '''
> +''''exec echo "Error: I can't find python anywhere"         # '''
> +#! /usr/bin/env python
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2017 Intel Corporation
> +#
> +
> +import copy
> +import getopt
> +import os
> +import platform
> +import subprocess
> +import sys
> +import time
> +from collections import OrderedDict
> +from os.path import exists, abspath, dirname, basename
> +from os import listdir
> +from os import system
> +from shutil import rmtree
> +from time import sleep
> +from math import ceil
> +
> +script_dir_path = os.getcwd()
> +
> +# The PCI base class for NETWORK devices
> +NETWORK_BASE_CLASS = "02"
> +
> +# global dict ethernet devices present. Dictionary indexed by PCI address.
> +# Each device within this is itself a dictionary of device properties
> +devices = {}
> +# list of supported DPDK drivers
> +dpdk_drivers = ["igb_uio", "vfio-pci", "uio_pci_generic"]
> +
> +# DPDK URL
> +dpdk_url = "http://dpdk.org/git/dpdk"
> +
> +dpdk_dir = "dpdk_demo"
> +
> +log_name = "/tmp/dpdk-quickstart.log"
> +
> +# command-line arg flags
> +b_flag = None
> +status_flag = False
> +force_flag = False
> +args = []
> +
> +
> +class NotRootException(Exception):
> +    pass
> +
> +
> +def log_output(args):
> +    with open(log_name, 'a') as log_file:
> +        log_file.write(args + '\n')
> +
> +
> +# This is roughly compatible with check_output function in subprocess module
> +# which is only available in python 2.7.
> +def check_output(args, stderr=None):
> +    '''Run a command and capture its output'''
> +    return subprocess.Popen(args, stdout=subprocess.PIPE,
> +                            stderr=subprocess.STDOUT).communicate()[0]
> +
> +
> +def check_output_dlg(args, stderr=None, title="console"):
> +    '''Run a command and capture its output'''
> +    p = subprocess.Popen(args, stdout=subprocess.PIPE,
> +                         stderr=subprocess.STDOUT)
> +    dlg.progressbox(fd=p.stdout.fileno(), text=title, width=78, height=20)
> +    return p.communicate()[0]
> +
> +
> +def error_out(msg):
> +    status = system(
> +              'dialog --backtitle Prerequisites --msgbox '
> +              '"{}" 10 80 2>/dev/null'.format(msg)
> +           )
> +    if status != 0:
> +        print("Error: {}".format(msg))
> +
> +
> +def python_ver():
> +    '''
> +        This function returns version of python used to run this script
> +    '''
> +    if sys.hexversion > 0x3000000:
> +        return 3
> +    elif sys.hexversion > 0x1000000:
> +        return 2
> +    else:
> +        return 1
> +
> +
> +def get_os():
> +    if platform.system() == 'Linux':
> +        # New in version 2.6 (< 2.6 -> platform.dist)
> +        return platform.linux_distribution()[0]
> +    else:
> +        return platform.system()
> +
> +
> +def get_arch():
> +    return platform.machine()
> +
> +
> +def get_target(os, arch):
> +    pass
> +
> +
> +def get_tool(tools):
> +    for tool in tools:
> +        if system("which {} > /dev/null 2>&1".format(tool)) == 0:
> +            return tool
> +
> +
> +def check_root():
> +    '''
> +        This function checks if script is run as root.
> +    '''
> +    return os.getuid() == 0
> +
> +
> +def check_dependencies(dependencies, system_data):
> +    installed = []
> +    pkg_check = {'Fedora': 'dnf list installed', 'Ubuntu': 'apt list'}
> +
> +    for dep in dependencies:
> +        output = subprocess.Popen("%s %s" % (pkg_check[system_data['os']],
> +                                  dependencies[dep]),
> +                                  shell=True,
> +                                  stdout=subprocess.PIPE,
> +                                  stderr=subprocess.PIPE)
> +        output = str(output.stdout.read())
> +        pkg_name = dependencies[dep]
> +        if system_data['os'] == 'Ubuntu':
> +                pkg_name = 'installed'
> +        elif 'uname' in pkg_name:
> +                pkg_name = pkg_name.split("$")[0]
> +                pkg_name = pkg_name.split("`")[0]
> +                pkg_name = pkg_name.strip("-")
> +                pkg_name = pkg_name.strip('"')
> +        # on ubuntu check for installed, on fedora for package name.
> +        if pkg_name in output:
> +            installed.append(dep)
> +    for package in installed:
> +        del dependencies[package]
> +    return dependencies
> +
> +
> +def install_dependencies():
> +    if not check_root():
> +        raise NotRootException()
> +    try:
> +        PKG_MGR = OrderedDict()
> +        PKG_MGR['dnf'] = ['install', '--best', '--allowerasing', '-y']
> +        PKG_MGR['apt-get'] = ['install', '-y']
> +        PKG_MGR['yum'] = ['install']
> +
> +        CMPLRS = OrderedDict()
> +        CMPLRS['gcc'] = []
> +        CMPLRS['clang'] = []
> +        CMPLRS['icc'] = []
> +
> +        headers = {'Fedora': 'kernel', 'Ubuntu': 'linux'}
> +
> +        system_data = {
> +            'os': get_os(),
> +            'arch': get_arch(),
> +            'pkg': get_tool(PKG_MGR),
> +            'compiler': get_tool(CMPLRS),
> +            'python': python_ver()
> +        }
> +
> +        dependencies = {
> +            'Fedora': {
> +                'git': 'git',
> +                'libc': 'glibc',
> +                'kernel-modules': '{}-modules'.format(
> +                                         headers[system_data['os']]),
> +                'kernel-devel': '"{}-devel-$(uname -r)"'.format(
> +                                         headers[system_data['os']]),
> +                'libnuma-devel': 'numactl-devel',
> +            },
> +            'Ubuntu': {
> +                'git': 'git',
> +                'libc': 'build-essential',
> +                'kernel-headers': '{}-headers-`uname -r`'.format(
> +                                         headers[system_data['os']]),
> +                'libnuma-devel': 'libnuma-dev',
> +            }
> +        }
> +
> +        dependencies = dependencies[system_data['os']]
> +        dependencies['coreutils'] = 'coreutils'
> +        if not system_data['compiler']:
> +            dependencies['compiler'] = 'gcc'
> +        dep_list = copy.copy(dependencies)
> +        dependencies = check_dependencies(dependencies, system_data)
> +        input_ = None
> +        reply = None
> +        try:
> +            import dialog
> +            input_ = dialog.Dialog(dialog="dialog", autowidgetsize=True)
> +            input_.set_background_title("Resolving dependecies")
> +            reply = input_.OK.upper()
> +            input_ = input_.yesno
> +        except ImportError:
> +            dependencies['dialog'] = 'python{}-dialog'.format(
> +                system_data['python'] if system_data['python'] == 3 else "")
> +        if system_data['python'] == 2 and not input_:
> +            input_ = raw_input
> +        elif system_data['python'] == 3 and not input_:
> +            input_ = input
> +        if len(dependencies) == 0:
> +            return
> +        prompt = "We are about to install following dependencies.\n"
> +        for key in dependencies:
> +            prompt += "{}\n".format(key)
> +        prompt += "Do you want to continue? (Y/n)\n"
> +        response = input_(prompt).upper()
> +        if response == "":
> +            response = "Y"
> +        if response != reply:
> +            prompt = "WARNING: Not installing packages. "
> +            prompt += "Assuming they have been manually installed. Proceed? (Y/n)"
> +            response = input_(prompt).upper()
> +            if response == "":
> +                response = "Y"
> +            if response != reply:
> +                sys.exit(1)
> +            else:
> +                return
> +        dlg = 'dialog' not in dependencies
> +        if dlg:
> +            dlg = dialog.Dialog(dialog='dialog')
> +        command = ""
> +        for dependency in dependencies:
> +            command += "{} {} {} {} &&".format(
> +                 system_data['pkg'],
> +                 PKG_MGR[system_data['pkg']][0],
> +                 dependencies[dependency],
> +                 " ".join(
> +                     PKG_MGR[system_data['pkg']][1:]) if len(
> +                                            PKG_MGR[system_data['pkg']]
> +                                        ) > 0 else "")
> +        log_output("# Installing dependencies")
> +        log_output(command)
> +        command += 'echo " "'
> +        output = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE,
> +                                  stderr=subprocess.PIPE)
> +        if dlg:
> +            dlg.progressbox(fd=output.stdout.fileno(),
> +                            text='Installing dependencies')
> +            output.wait()
> +        else:
> +            while output.poll() is None:
> +                print(str(output.stdout.readline()).strip("b'\\n\n"))
> +                print(str(output.stderr.readline()).strip("b'\\n\n"))
> +                sys.stdout.flush()
> +            print("Press any key to continue...")
> +            ch = sys.stdin.read(1)
> +        not_installed = check_dependencies(dep_list, system_data)
> +        if len(not_installed) > 0:
> +            raise Exception(str(not_installed))
> +
> +    except Exception as e:
> +        lst = ":\n" + str(e).strip("{}") if str(e) else "."
> +        status = system(
> +              'dialog --backtitle Prerequisites --msgbox '
> +              '"Unable to install dependencies%s"'
> +              ' 10 80 2>/dev/null' % lst
> +           )
> +        if status != 0:
> +            print("Error: unable to install dependecies%s" % lst)
> +
> +
> +def find_module(mod):
> +
> +    mod = mod + ".ko"
> +    paths = check_output(["find", ".", "-name", mod])
> +    path = paths.decode().splitlines()[0]
> +
> +    if exists(path):
> +        return path
> +
> +    '''find the .ko file for kernel module named mod.
> +    Searches the $RTE_SDK/$RTE_TARGET directory, the kernel
> +    modules directory and finally under the parent directory of
> +    the script '''
> +    # check $RTE_SDK/$RTE_TARGET directory
> +    if 'RTE_SDK' in os.environ and 'RTE_TARGET' in os.environ:
> +        path = "%s/%s/kmod/%s.ko" % (os.environ['RTE_SDK'],
> +                                     os.environ['RTE_TARGET'], mod)
> +        if exists(path):
> +            return path
> +
> +    # check using depmod
> +    try:
> +        depmod_out = check_output(["modinfo", "-n", mod],
> +                                  stderr=subprocess.STDOUT).lower()
> +        if "error" not in depmod_out:
> +            path = depmod_out.strip()
> +            if exists(path):
> +                return path
> +    except:  # if modinfo can't find module, it fails, so continue
> +        pass
> +
> +    # check for a copy based off current path
> +    tools_dir = dirname(abspath(sys.argv[0]))
> +    if tools_dir.endswith("tools"):
> +        base_dir = dirname(tools_dir)
> +        find_out = check_output(["find", base_dir, "-name", mod + ".ko"])
> +        if len(find_out) > 0:  # something matched
> +            path = find_out.splitlines()[0]
> +            if exists(path):
> +                return path
> +
> +
> +def check_modules():
> +    '''Checks that igb_uio is loaded'''
> +    global dpdk_drivers
> +
> +    os.system("modprobe uio")
> +
> +    # Insert igb_uio module if not already inserted
> +    out = check_output(["lsmod"])
> +    if "igb_uio" not in out.decode():
> +        try:
> +            path = find_module("igb_uio")
> +            os.system("insmod " + path)
> +        except:
> +            dlg.msgbox("Error - igb_uio module was not found",
> +                       height=None, width=None)
> +            sys.exit(1)
> +            return
> +
> +    # list of supported modules
> +    mods = [{"Name": driver, "Found": False} for driver in dpdk_drivers]
> +
> +    # first check if module is loaded
> +    try:
> +        # Get list of sysfs modules (both built-in and dynamically loaded)
> +        sysfs_path = '/sys/module/'
> +
> +        # Get the list of directories in sysfs_path
> +        sysfs_mods = [os.path.join(sysfs_path, o) for o
> +                      in os.listdir(sysfs_path)
> +                      if os.path.isdir(os.path.join(sysfs_path, o))]
> +
> +        # Extract the last element of '/sys/module/abc' in the array
> +        sysfs_mods = [a.split('/')[-1] for a in sysfs_mods]
> +
> +        # special case for vfio_pci (module is named vfio-pci,
> +        # but its .ko is named vfio_pci)
> +        sysfs_mods = map(lambda a:
> +                         a if a != 'vfio_pci' else 'vfio-pci', sysfs_mods)
> +
> +        for mod in mods:
> +            if mod["Name"] in sysfs_mods:
> +                mod["Found"] = True
> +    except:
> +        pass
> +
> +    # check if we have at least one loaded module
> +    if True not in [mod["Found"] for mod in mods] and b_flag is not None:
> +        if b_flag in dpdk_drivers:
> +            dlg.msgbox("Error - no supported modules (DPDK driver) are loaded",
> +                       height=None, width=None)
> +            sys.exit(1)
> +        else:
> +            dlg.msgbox("Warning - no supported modules (DPDK driver) "
> +                       " are loaded", height=None, width=None)
> +
> +    # change DPDK driver list to only contain drivers that are loaded
> +    dpdk_drivers = [mod["Name"] for mod in mods if mod["Found"]]
> +
> +
> +def has_driver(dev_id):
> +    '''return true if a device is assigned to a driver. False otherwise'''
> +    return "Driver_str" in devices[dev_id]
> +
> +
> +def get_pci_device_details(dev_id):
> +    '''This function gets additional details for a PCI device'''
> +    device = {}
> +
> +    extra_info = check_output(["lspci", "-vmmks", dev_id]).splitlines()
> +
> +    # parse lspci details
> +    for line in extra_info:
> +        if len(line) == 0:
> +            continue
> +        name, value = line.decode().split("\t", 1)
> +        name = name.strip(":") + "_str"
> +        device[name] = value
> +    # check for a unix interface name
> +    device["Interface"] = ""
> +    for base, dirs, _ in os.walk("/sys/bus/pci/devices/%s/" % dev_id):
> +        if "net" in dirs:
> +            device["Interface"] = \
> +                ",".join(os.listdir(os.path.join(base, "net")))
> +            break
> +    # check if a port is used for ssh connection
> +    device["Ssh_if"] = False
> +    device["Active"] = ""
> +
> +    return device
> +
> +
> +def bind_nic_ports():
> +    get_nic_details()
> +    return nic_bind_dialog()
> +
> +
> +def get_nic_details():
> +    '''This function populates the "devices" dictionary. The keys used are
> +    the pci addresses (domain:bus:slot.func). The values are themselves
> +    dictionaries - one for each NIC.'''
> +    global devices
> +    global dpdk_drivers
> +
> +    # clear any old data
> +    devices = {}
> +    # first loop through and read details for all devices
> +    # request machine readable format, with numeric IDs
> +    dev = {}
> +    dev_lines = check_output(["lspci", "-Dvmmn"]).splitlines()
> +    for dev_line in dev_lines:
> +        if len(dev_line) == 0:
> +            if dev["Class"][0:2] == NETWORK_BASE_CLASS:
> +                # convert device and vendor ids to numbers, then add to global
> +                dev["Vendor"] = int(dev["Vendor"], 16)
> +                dev["Device"] = int(dev["Device"], 16)
> +                # use dict to make copy of dev
> +                devices[dev["Slot"]] = dict(dev)
> +        else:
> +            name, value = dev_line.decode().split("\t", 1)
> +            dev[name.rstrip(":")] = value
> +
> +    # check what is the interface if any for an ssh connection if
> +    # any to this host, so we can mark it later.
> +    ssh_if = []
> +    route = check_output(["ip", "-o", "route"])
> +    # filter out all lines for 169.254 routes
> +    route = "\n".join(filter(lambda ln: not ln.startswith("169.254"),
> +                             route.decode().splitlines()))
> +    rt_info = route.split()
> +    for i in range(len(rt_info) - 1):
> +        if rt_info[i] == "dev":
> +            ssh_if.append(rt_info[i+1])
> +
> +    # based on the basic info, get extended text details
> +    for d in devices:
> +        # get additional info and add it to existing data
> +        devices[d] = devices[d].copy()
> +        devices[d].update(list(get_pci_device_details(d).items()))
> +
> +        for _if in ssh_if:
> +            if _if in devices[d]["Interface"].split(","):
> +                devices[d]["Ssh_if"] = True
> +                devices[d]["Active"] = "*Active*"
> +                break
> +
> +        # add igb_uio to list of supporting modules if needed
> +        if "Module_str" in devices[d]:
> +            for driver in dpdk_drivers:
> +                if driver not in devices[d]["Module_str"]:
> +                    devices[d]["Module_str"] = \
> +                        devices[d]["Module_str"] + ",%s" % driver
> +        else:
> +            devices[d]["Module_str"] = ",".join(dpdk_drivers)
> +
> +        # make sure the driver and module strings do not have any duplicates
> +        if has_driver(d):
> +            modules = devices[d]["Module_str"].split(",")
> +            if devices[d]["Driver_str"] in modules:
> +                modules.remove(devices[d]["Driver_str"])
> +                devices[d]["Module_str"] = ",".join(modules)
> +
> +
> +def dev_id_from_dev_name(dev_name):
> +    '''Take a device "name" - a string passed in by user to identify a NIC
> +    device, and determine the device id - i.e. the domain:bus:slot.func - for
> +    it, which can then be used to index into the devices array'''
> +
> +    # check if it's already a suitable index
> +    if dev_name in devices:
> +        return dev_name
> +    # check if it's an index just missing the domain part
> +    elif "0000:" + dev_name in devices:
> +        return "0000:" + dev_name
> +    else:
> +        # check if it's an interface name, e.g. eth1
> +        for d in devices:
> +            if dev_name in devices[d]["Interface"].split(","):
> +                return devices[d]["Slot"]
> +    # if nothing else matches - error
> +    dlg.msgbox("Unknown device: %s. "
> +               "Please specify device in \"bus:slot.func\" format" % dev_name,
> +               height=None, width=None)
> +    sys.exit(1)
> +
> +
> +def unbind_one(dev_id, force):
> +    '''Unbind the device identified by "dev_id" from its current driver'''
> +    dev = devices[dev_id]
> +    if not has_driver(dev_id):
> +        dlg.msgbox("%s %s %s is not currently managed by any driver\n" %
> +                   (dev["Slot"], dev["Device_str"], dev["Interface"]),
> +                   height=None, width=None)
> +        return
> +
> +    # prevent us disconnecting ourselves
> +    if dev["Ssh_if"] and not force:
> +        dlg.msgbox("Routing table indicates that interface %s is active. "
> +                   "Skipping unbind" % (dev_id), height=None, width=None)
> +        return
> +
> +    # write to /sys to unbind
> +    filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"]
> +    try:
> +        f = open(filename, "a")
> +    except:
> +        dlg.msgbox("Error: unbind failed for %s - Cannot open %s"
> +                   % (dev_id, filename), height=None, width=None)
> +        sys.exit(1)
> +    log_output("echo " + dev_id + " > " + filename)
> +    f.write(dev_id)
> +    f.close()
> +
> +
> +def bind_one(dev_id, driver, force):
> +    '''Bind the device given by "dev_id" to the driver "driver". If the device
> +    is already bound to a different driver, it will be unbound first'''
> +    dev = devices[dev_id]
> +    saved_driver = None  # used to rollback any unbind in case of failure
> +
> +    # prevent disconnection of our ssh session
> +    if dev["Ssh_if"] and not force:
> +        dlg.msgbox("Routing table indicates that interface %s is active. "
> +                   "Not modifying" % (dev_id), height=None, width=None)
> +        return
> +
> +    # unbind any existing drivers we don't want
> +    if has_driver(dev_id):
> +        if dev["Driver_str"] == driver:
> +            return
> +        else:
> +            saved_driver = dev["Driver_str"]
> +            unbind_one(dev_id, force)
> +            dev["Driver_str"] = ""  # clear driver string
> +
> +    # if we are binding to one of DPDK drivers, add PCI id's to that driver
> +    if driver in dpdk_drivers:
> +        filename = "/sys/bus/pci/drivers/%s/new_id" % driver
> +        device_id = "%04x %04x" % (dev["Vendor"], dev["Device"])
> +        try:
> +            f = open(filename, "w")
> +        except:
> +            dlg.msgbox("Error: bind failed for %s - Cannot open %s"
> +                       % (dev_id, filename), height=None, width=None)
> +            return
> +        try:
> +            log_output("echo " + device_id + " > " + filename)
> +            f.write(device_id)
> +            f.close()
> +        except:
> +            dlg.msgbox("Error: bind failed for %s - Cannot write "
> +                       "new PCI ID to driver %s" % (dev_id, driver),
> +                       height=None, width=None)
> +            return
> +
> +    # do the bind by writing to /sys
> +    filename = "/sys/bus/pci/drivers/%s/bind" % driver
> +    try:
> +        f = open(filename, "a")
> +    except:
> +        dlg.msgbox("Error: bind failed for %s - Cannot open %s"
> +                   % (dev_id, filename), height=None, width=None)
> +        if saved_driver is not None:  # restore any previous driver
> +            bind_one(dev_id, saved_driver, force)
> +        return
> +    try:
> +        log_output("echo " + dev_id + " > " + filename)
> +        f.write(dev_id)
> +        f.close()
> +    except:
> +        # for some reason, closing dev_id after adding a new PCI ID to new_id
> +        # results in IOError. however, if the device was successfully bound,
> +        # we don't care for any errors and can safely ignore IOError
> +        tmp = get_pci_device_details(dev_id)
> +        if "Driver_str" in tmp and tmp["Driver_str"] == driver:
> +            return
> +        dlg.msgbox("Error: bind failed for %s - Cannot bind to driver %s"
> +                   % (dev_id, driver), height=None, width=None)
> +        if saved_driver is not None:  # restore any previous driver
> +            bind_one(dev_id, saved_driver, force)
> +        return
> +
> +
> +def unbind_nics(dev_list, force=False):
> +    """Unbind method, takes a list of device locations"""
> +    dev_list = map(dev_id_from_dev_name, dev_list)
> +    for d in dev_list:
> +        unbind_one(d, force)
> +
> +
> +def bind_nics(dev_list, driver, force=False):
> +    """Bind method, takes a list of device locations"""
> +    global devices
> +
> +    dev_list = list(map(dev_id_from_dev_name, dev_list))
> +    for d in dev_list:
> +        bind_one(d, driver, force)
> +
> +    # when binding devices to a generic driver (i.e. one that doesn't have a
> +    # PCI ID table), some devices that are not bound to any other driver could
> +    # be bound even if no one has asked them to. hence, we check the list of
> +    # drivers again, and see if some of the previously-unbound devices were
> +    # erroneously bound.
> +    for d in devices:
> +        # skip devices that were already bound or that we know should be bound
> +        if "Driver_str" in devices[d] or d in dev_list:
> +            continue
> +        # update information about this device
> +        devices[d] = dict((list(devices[d].items())) +
> +                          (list(get_pci_device_details(d).items())))
> +
> +        # check if updated information indicates that the device was bound
> +        if "Driver_str" in devices[d]:
> +            unbind_one(d, force)
> +
> +
> +def nic_bind_dialog():
> +    '''Function called when the script is passed the "--status" option.
> +    Displays to the user what devices are bound to the igb_uio driver, the
> +    kernel driver or to no driver'''
> +    global dpdk_drivers
> +    kernel_drv = []
> +    dpdk_drv = []
> +    no_drv = []
> +    choices = []
> +
> +    # split our list of network devices into the three categories above
> +    for d in devices:
> +        if NETWORK_BASE_CLASS in devices[d]["Class"]:
> +            dev = devices[d]
> +            if not has_driver(d):
> +                no_drv.append(devices[d])
> +                choices.append((dev["Slot"], dev["Device_str"], False))
> +                continue
> +            if devices[d]["Driver_str"] in dpdk_drivers:
> +                dpdk_drv.append(devices[d])
> +                choices.append((dev["Slot"], dev["Device_str"], True))
> +            else:
> +                kernel_drv.append(devices[d])
> +                choices.append((dev["Slot"], dev["Device_str"], False))
> +
> +    choices.sort()
> +    code, tags = dlg.checklist("Select two loopback ports to bind to DPDK "
> +                               "driver (<space> to select/deselect)",
> +                               choices=choices, extra_button=True,
> +                               ok_label='Use ports selected',
> +                               extra_label='Detect loopback',
> +                               cancel_label='Use Virtual ports')
> +
> +    if code == Dialog.OK:
> +        b_list = []
> +        u_list = []
> +        for tag in tags:
> +            for k_drv in kernel_drv:
> +                if k_drv["Slot"] == tag:
> +                    b_list.append(tag)
> +            for n_drv in no_drv:
> +                if n_drv["Slot"] == tag:
> +                    b_list.append(tag)
> +        for d_drv in dpdk_drv:
> +            if d_drv["Slot"] not in tags:
> +                u_list.append(d_drv["Slot"])
> +
> +        bind_nics(b_list, 'igb_uio')
> +        unbind_nics(u_list)
> +        if (len(tags) != 2):
> +            dlg.msgbox("Select two ports from the list or use virtual ports",
> +                       height=5, width=60)
> +            return bind_nic_ports()
> +        return 1
> +
> +    # Detect loopback
> +    if code == Dialog.EXTRA:
> +        dlg.msgbox("About to detect loopback... Press enter to start...",
> +                   height=5, width=60)
> +        dlg.infobox("Detecting loopback... this might take several seconds",
> +                    height=5, width=60)
> +        for d in dpdk_drv:
> +            drivers_unused = d["Module_str"].split(",")
> +            for driver in drivers_unused:
> +                if driver in dpdk_drivers:
> +                    continue
> +                else:
> +                    bind_one(d["Slot"], driver, False)
> +        get_nic_details()
> +        if detect_loopback() == 0:
> +            # Bind everything back
> +            for d in dpdk_drv:
> +                bind_one(d['Slot'], 'igb_uio', False)
> +
> +        return bind_nic_ports()
> +
> +    # Use Virtual PMDs
> +    if code == Dialog.CANCEL:
> +        return 0
> +
> +
> +def detect_loopback():
> +    ports = []
> +    ports_link_up = []
> +
> +    # Get new list of network devices bound to the kernel driver
> +    for d in devices:
> +        if NETWORK_BASE_CLASS in devices[d]["Class"]:
> +            dev = devices[d]
> +            if has_driver(d) and dev["Driver_str"] not in dpdk_drivers:
> +                # Do not use a device with an active connection
> +                if not dev["Ssh_if"]:
> +                    ports.append(dev)
> +
> +    # Need at least two ports
> +    if len(ports) < 2:
> +        dlg.msgbox("At least two ports bound to the kernel driver are needed",
> +                   height=5, width=60)
> +        return 0
> +
> +    # Bring up all interfaces
> +    for dev in ports:
> +        iface = dev["Interface"]
> +        ifconfig_out = check_output(["ifconfig", iface])
> +        dev["Status"] = "UP" in str(ifconfig_out)
> +        if not dev["Status"]:
> +            os.system("ifconfig " + iface + " up")
> +
> +    # Sleep for 3 seconds to give time for the link status to be correctly set
> +    time.sleep(3)
> +    # Check link statuses
> +    for dev in ports:
> +        iface = dev["Interface"]
> +        ifconfig_out = check_output(["ifconfig", iface])
> +        dev["Link"] = "RUNNING" in str(ifconfig_out)
> +        if dev["Link"]:
> +            ports_link_up.append(dev)
> +
> +    # Need at least two ports with the link status up
> +    if len(ports_link_up) < 2:
> +        dlg.msgbox("At least two ports with the link status up are required",
> +                   height=5, width=60)
> +        restore_interfaces(ports)
> +        return 0
> +
> +    # Check for link status changes when bringing down ports,
> +    # to find a loopback connection
> +    for i in range(len(ports_link_up)):
> +        # Bring down interface
> +        dev_down = ports_link_up[i]
> +        iface_down = dev_down["Interface"]
> +        os.system("ifconfig " + iface_down + " down")
> +        time.sleep(3)
> +        for j in range(i + 1, len(ports_link_up)):
> +            dev_test = ports_link_up[j]
> +            iface_test = dev_test["Interface"]
> +            ifconfig_out = check_output(["ifconfig", iface_test])
> +            new_link_status = "RUNNING" in str(ifconfig_out)
> +            if not new_link_status:
> +                # Found a loopback connection, since the link went down
> +                dlg.msgbox("Loopback detected! (Ports will be pre-selected "
> +                           "in the next page)",
> +                           height=6, width=60)
> +
> +                # Restore the other ports
> +                restore_interfaces(ports)
> +
> +                # Bind both interfaces
> +                bind_one(dev_test['Slot'], 'igb_uio', False)
> +                bind_one(dev_down['Slot'], 'igb_uio', False)
> +
> +                return 1
> +
> +        # Bring up interface
> +        os.system("ifconfig " + iface_test + " up")
> +        time.sleep(3)
> +
> +    dlg.msgbox("No loopback could be detected",
> +               height=5, width=60)
> +
> +    restore_interfaces(ports)
> +    return 0
> +
> +
> +def restore_interfaces(ports):
> +    for dev in ports:
> +        # Port was down before, so bring it down again
> +        if not dev["Status"]:
> +            iface = dev["Interface"]
> +            os.system("ifconfig " + iface + "down")
> +
> +
> +def clone_dpdk():
> +
> +    if os.path.exists(dpdk_dir):
> +        ret = dlg.yesno("Directory [" + dpdk_dir + "] already exists.\n"
> +                        "Continue by compiling code in existing directory?",
> +                        height=6, width=70, extra_button=True,
> +                        extra_label='Delete')
> +        if (ret == 'cancel'):
> +            print("\nNot using existing directory. Exiting.")
> +            sys.exit(0)
> +        elif (ret == Dialog.EXTRA):
> +            rmtree(dpdk_dir, ignore_errors=True)
> +            clone_dpdk()
> +    else:
> +        try:
> +            title_str = "Cloning DPDK repository from dpdk.org..."
> +            log_output("# Cloning DPDK")
> +            log_output("git clone " + dpdk_url + " " + dpdk_dir)
> +            clone_out = check_output_dlg(["git", "clone", dpdk_url, dpdk_dir],
> +                                         title=title_str)
> +            dpdk_path = script_dir_path + "/" + dpdk_dir
> +            os.chdir(dpdk_path)
> +
> +            # Get last DPDK release tag #
> +            tag_out = check_output(["git", "tag", "--sort", "version:refname"])
> +            tag_list = tag_out.split()
> +            tag = tag_list.pop().decode()
> +
> +            while "rc" in tag:
> +                tag = tag_list.pop().decode()
> +
> +            log_output("git checkout " + tag)
> +            checkout_out = check_output(["git", "checkout", tag])
> +
> +        except:
> +            dlg.msgbox("Failed to check out source from dpdk.org",
> +                       height=5, width=60)
> +            sys.exit(1)
> +
> +
> +def compile_dpdk():
> +
> +    dpdk_path = script_dir_path + "/" + dpdk_dir
> +    os.chdir(dpdk_path)
> +    log_output("# Compiling DPDK")
> +    log_output("make defconfig")
> +    check_output(["make", "defconfig"])
> +    ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
> +    if ncpus > 1:
> +        ncpus = ncpus - 1
> +    log_output('make -j ' + str(ncpus))
> +    compilation_out = check_output_dlg(["make", "-j", str(ncpus)],
> +                                       title="Compiling DPDK...")
> +    if b"Error" in compilation_out:
> +        print("Compilation failed")
> +        sys.exit(1)
> +
> +
> +def setup_hugepages():
> +    numa_out = check_output(["lscpu"])
> +    output_dict = dict(line.decode().split(":") for line in numa_out.splitlines())
> +    numa_nodes = int(output_dict["NUMA node(s)"].strip())
> +
> +    # Minimum memory required is 64 MB per node
> +    min_mem_node = 64 * 1024
> +
> +    # Get hugepage size
> +    meminfo_out = check_output(["cat", "/proc/meminfo"])
> +    output_dict = dict(line.decode().split(":") for line in meminfo_out.splitlines())
> +    hugepagesz = int(output_dict['Hugepagesize'].split()[0])
> +
> +    # Get minimum number of pages
> +    min_pages = int(ceil(float(min_mem_node) / hugepagesz))
> +    hfile = "/sys/kernel/mm/hugepages/hugepages-" + str(hugepagesz) + \
> +            "kB/nr_hugepages"
> +
> +    # Check number of pages available
> +    with open(hfile, 'r') as myfile:
> +        data = myfile.read().replace('\n', '')
> +    num_pages = int(data)
> +
> +    # If there are not enough hugepages, ask the user to reserve some
> +    if (num_pages >= min_pages):
> +        return
> +
> +    while(1):
> +        title = "Number of Hugepages (minimum required displayed)"
> +        data = dlg.inputbox(title, height=None, width=None,
> +                            init=str(min_pages))
> +        if (data[0] == 'ok'):
> +            if (int(data[1]) < min_pages):
> +                dlg.msgbox("Not enough hugepages",
> +                           height=None, width=None)
> +                continue
> +
> +            log_output("# Setting up hugepages:")
> +            log_output("echo " + str(data[1]) + " > " + hfile)
> +            with open(hfile, 'w') as myfile:
> +                ret = myfile.write(data[1])
> +                return
> +
> +
> +def run_testpmd(use_physical_pmds):
> +
> +    testpmd = ""
> +
> +    paths = check_output(["find", ".", "-name", "testpmd"])
> +    path = paths.decode().splitlines()[0]
> +
> +    for path in paths.split():
> +        testpmd = path.decode()
> +    if (use_physical_pmds == 1):
> +        testpmd_args = (
> +            ' -l 0,1 -- --stats-period 1 --tx-first'
> +            ' --no-lsc-interrupt --total-num-mbufs 8192'
> +        )
> +    else:
> +        testpmd_args = (
> +            ' -l 0,1 --vdev net_ring0 --vdev net_ring1 --no-pci --'
> +            ' --stats-period 1 --tx-first'
> +            ' --no-lsc-interrupt --total-num-mbufs 8192'
> +        )
> +
> +    testpmd_str = testpmd + testpmd_args
> +    log_output("# Running TestPMD")
> +    log_output(testpmd_str)
> +    testpmd_list = testpmd_str.split()
> +    subprocess.call(testpmd_list)
> +
> +
> +def main():
> +    '''program main function'''
> +    if not check_root():
> +        raise NotRootException()
> +    if "-b" in sys.argv:
> +        check_modules()
> +        bind_nic_ports()
> +        return
> +    if "-p" in sys.argv:
> +        setup_hugepages()
> +        return
> +    if "-d" in sys.argv:
> +        return
> +    if "-c" in sys.argv:
> +        clone_dpdk()
> +        return
> +    if "-m" in sys.argv:
> +        clone_dpdk()
> +        compile_dpdk()
> +        return
> +
> +    clone_dpdk()
> +    compile_dpdk()
> +
> +    log_output("# Probing DPDK driver igb_uio")
> +    log_output("modprobe uio")
> +    os.system("modprobe uio")
> +
> +    try:
> +        path = find_module("igb_uio")
> +        log_output("insmod " + path)
> +        os.system("insmod " + path)
> +    except:
> +        print("igb_uio not found")
> +        return
> +
> +    setup_hugepages()
> +
> +    check_modules()
> +    log_output("# Binding ports to DPDK driver")
> +    use_physical_nic = bind_nic_ports()
> +    dlg.msgbox("About to run testpmd DPDK application...", height=5, width=60)
> +    os.system("clear")
> +    run_testpmd(use_physical_nic)
> +
> +try:
> +    install_dependencies()
> +    from dialog import Dialog
> +    dlg = Dialog(dialog="dialog", autowidgetsize=True)
> +    dlg.set_background_title("DPDK Setup and Installation Script")
> +    if __name__ == "__main__":
> +        main()
> +except NotRootException:
> +    error_out("You need to run this script as Root")
> +except KeyboardInterrupt:
> +    dlg.msgbox("Log file saved in " + log_name + "\nPress enter to exit")
> +    sys.exit(0)