[dts] [‘dts-v1’ 3/9] Add some params and functions related to the virtual test

[sjiajiax <sunx.jiajia@intel.com>]

Signed-off-by: sjiajiax <sunx.jiajia@intel.com>
---
 framework/dts.py          | 92 +++++++++++------------------------------------
 framework/exception.py    | 27 ++++++++++++++
 framework/logger.py       | 69 +++++++++++++++++++++++++++++------
 framework/main.py         |  7 +++-
 framework/project_dpdk.py | 62 ++++++++++++++++++++++----------
 5 files changed, 157 insertions(+), 100 deletions(-)

diff --git a/framework/dts.py b/framework/dts.py
index c9ecccb..c0df4e9 100644
--- a/framework/dts.py
+++ b/framework/dts.py
@@ -49,6 +49,7 @@ from test_case import TestCase
 from test_result import Result
 from stats_reporter import StatsReporter
 from excel_reporter import ExcelReporter
+from utils import *
 from exception import TimeoutException
 from logger import getLogger
 import logger
@@ -57,6 +58,7 @@ import sys
 reload(sys)
 sys.setdefaultencoding('UTF8')
 
+PROJECT_MODULE_PREFIX = 'project_'
 
 debug_mode = False
 config = None
@@ -73,44 +75,12 @@ result = None
 excel_report = None
 stats = None
 log_handler = None
+Package = ''
+Patches = []
 drivername = ""
 interrupttypr = ""
 
 
-def RED(text):
-    return "\x1B[" + "31;1m" + text + "\x1B[" + "0m"
-
-
-def BLUE(text):
-    return "\x1B[" + "36;1m" + text + "\x1B[" + "0m"
-
-
-def GREEN(text):
-    return "\x1B[" + "32;1m" + text + "\x1B[" + "0m"
-
-
-def regexp(s, to_match, allString=False):
-    """
-    Ensure that the re `to_match' only has one group in it.
-    """
-
-    scanner = re.compile(to_match, re.DOTALL)
-    if allString:
-        return scanner.findall(s)
-    m = scanner.search(s)
-    if m is None:
-        log_handler.warning("Failed to match " + to_match + " in the string " + s)
-        return None
-    return m.group(1)
-
-
-def pprint(some_dict):
-    """
-    Print JSON format dictionary object.
-    """
-    return json.dumps(some_dict, sort_keys=True, indent=4)
-
-
 def report(text, frame=False, annex=False):
     """
     Save report text into rst file.
@@ -132,36 +102,6 @@ def close_crb_sessions():
     log_handler.info("DTS ended")
 
 
-def get_nic_driver(pci_id):
-    """
-    Return linux driver for specified pci device
-    """
-    driverlist = dict(zip(NICS.values(), DRIVERS.keys()))
-    try:
-        driver = DRIVERS[driverlist[pci_id]]
-    except Exception as e:
-        driver = None
-    return driver
-
-
-def accepted_nic(pci_id):
-    """
-    Return True if the pci_id is a known NIC card in the settings file and if
-    it is selected in the execution file, otherwise it returns False.
-    """
-    if pci_id not in NICS.values():
-        return False
-
-    if nic is 'any':
-        return True
-
-    else:
-        if pci_id == NICS[nic]:
-            return True
-
-    return False
-
-
 def get_crb_os(crb):
     if 'OS' in crb:
         return crb['OS']
@@ -220,9 +160,10 @@ def get_project_obj(project_name, super_class, crbInst, serializer):
     """
     Load project module and return crb instance.
     """
+    global PROJECT_MODULE_PREFIX
     project_obj = None
     try:
-        project_module = __import__("project_" + project_name)
+        project_module = __import__(PROJECT_MODULE_PREFIX + project_name)
 
         for project_subclassname, project_subclass in get_subclasses(project_module, super_class):
             project_obj = project_subclass(crbInst, serializer)
@@ -267,19 +208,20 @@ def dts_log_execution(log_handler):
         pass
 
 
-def dts_crbs_init(crbInst, skip_setup, read_cache, project, base_dir, nic):
+def dts_crbs_init(crbInst, skip_setup, read_cache, project, base_dir, nic, virttype):
     """
     Create dts dut/tester instance and initialize them.
     """
     global dut
     global tester
-    serializer.set_serialized_filename('../.%s.cache' % crbInst['IP'])
+    serializer.set_serialized_filename('.%s.cache' % crbInst['IP'])
     serializer.load_from_file()
 
     dut = get_project_obj(project, Dut, crbInst, serializer)
     tester = get_project_obj(project, Tester, crbInst, serializer)
     dut.tester = tester
     tester.dut = dut
+    dut.set_virttype(virttype)
     dut.set_speedup_options(read_cache, skip_setup)
     dut.set_directory(base_dir)
     dut.set_nic_type(nic)
@@ -337,7 +279,6 @@ def dts_run_target(crbInst, targets, test_suites, nic):
         if 'nic_type' not in paramDict:
             paramDict['nic_type'] = 'any'
             nic = 'any'
-        result.nic = nic
 
         dts_run_suite(crbInst, test_suites, target, nic)
 
@@ -359,7 +300,9 @@ def dts_run_suite(crbInst, test_suites, target, nic):
             test_module = __import__('TestSuite_' + test_suite)
             for test_classname, test_class in get_subclasses(test_module, TestCase):
 
-                test_suite = test_class(dut, tester, target)
+                test_suite = test_class(dut, tester, target, test_suite)
+                result.nic = test_suite.nic
+
                 dts_log_testsuite(test_suite, log_handler, test_classname)
 
                 log_handler.info("\nTEST SUITE : " + test_classname)
@@ -386,7 +329,7 @@ def dts_run_suite(crbInst, test_suites, target, nic):
 
 def run_all(config_file, pkgName, git, patch, skip_setup,
             read_cache, project, suite_dir, test_cases,
-            base_dir, output_dir, verbose, debug):
+            base_dir, output_dir, verbose, virttype, debug):
     """
     Main process of DTS, it will run all test suites in the config file.
     """
@@ -400,6 +343,12 @@ def run_all(config_file, pkgName, git, patch, skip_setup,
     global stats
     global log_handler
     global debug_mode
+    global Package
+    global Patches
+
+    # save global variable
+    Package = pkgName
+    Patches = patch
 
     # prepare the output folder
     if not os.path.exists(output_dir):
@@ -466,7 +415,8 @@ def run_all(config_file, pkgName, git, patch, skip_setup,
         result.dut = dutIP
 
         # init dut, tester crb
-        dts_crbs_init(crbInst, skip_setup, read_cache, project, base_dir, nics)
+        dts_crbs_init(
+            crbInst, skip_setup, read_cache, project, base_dir, nics, virttype)
 
         # Run DUT prerequisites
         if dts_run_prerequisties(pkgName, patch) is False:
diff --git a/framework/exception.py b/framework/exception.py
index be38c16..98dedf4 100644
--- a/framework/exception.py
+++ b/framework/exception.py
@@ -46,3 +46,30 @@ class SSHConnectionException(Exception):
 
     def __str__(self):
         return 'Error trying to connect with %s' % self.host
+
+
+class SSHSessionDeadException(Exception):
+
+    """
+    SSH session is not alive.
+    It can no longer be used.
+    """
+
+    def __init__(self, host):
+        self.host = host
+
+    def __str__(self):
+        return 'SSH session with %s has been dead' % self.host
+
+
+class StartVMFailedException(Exception):
+
+    """
+    Start VM failed.
+    """
+
+    def __init__(self, error):
+        self.error = error
+
+    def __str__(self):
+        return repr(self.error)
diff --git a/framework/logger.py b/framework/logger.py
index 1829e18..5597e33 100644
--- a/framework/logger.py
+++ b/framework/logger.py
@@ -35,6 +35,9 @@ import sys
 import inspect
 import re
 
+from settings import LOG_NAME_SEP
+from utils import RED
+
 """
 DTS logger module with several log level. DTS framwork and TestSuite log
 will saved into different log files.
@@ -58,6 +61,9 @@ logging.SUITE_TESTER_OUTPUT = logging.DEBUG + 4
 logging.DTS_IXIA_CMD = logging.INFO + 5
 logging.DTS_IXIA_OUTPUT = logging.DEBUG + 5
 
+logging.DTS_VIRTDUT_CMD = logging.INFO + 6
+logging.DTS_VIRTDUT_OUTPUT = logging.DEBUG + 6
+
 logging.addLevelName(logging.DTS_DUT_CMD, 'DTS_DUT_CMD')
 logging.addLevelName(logging.DTS_DUT_OUTPUT, 'DTS_DUT_OUTPUT')
 logging.addLevelName(logging.DTS_DUT_RESULT, 'DTS_DUT_RESUTL')
@@ -66,6 +72,12 @@ logging.addLevelName(logging.DTS_TESTER_CMD, 'DTS_TESTER_CMD')
 logging.addLevelName(logging.DTS_TESTER_OUTPUT, 'DTS_TESTER_OUTPUT')
 logging.addLevelName(logging.DTS_TESTER_RESULT, 'DTS_TESTER_RESULT')
 
+logging.addLevelName(logging.DTS_IXIA_CMD, 'DTS_IXIA_CMD')
+logging.addLevelName(logging.DTS_IXIA_OUTPUT, 'DTS_IXIA_OUTPUT')
+
+logging.addLevelName(logging.DTS_VIRTDUT_CMD, 'VIRTDUT_CMD')
+logging.addLevelName(logging.DTS_VIRTDUT_OUTPUT, 'VIRTDUT_OUTPUT')
+
 logging.addLevelName(logging.SUITE_DUT_CMD, 'SUITE_DUT_CMD')
 logging.addLevelName(logging.SUITE_DUT_OUTPUT, 'SUITE_DUT_OUTPUT')
 
@@ -82,15 +94,18 @@ stream_fmt = '%(color)s%(levelname)20s: %(message)s' + RESET_COLOR
 log_dir = None
 
 
-def RED(text):
-    return "\x1B[" + "31;1m" + text + "\x1B[" + "0m"
-
-
 def set_verbose():
     global verbose
     verbose = True
 
 
+def add_salt(salt, msg):
+    if not salt:
+        return msg
+    else:
+        return '[%s] ' % salt + str(msg)
+
+
 class BaseLoggerAdapter(logging.LoggerAdapter):
     """
     Upper layer of original logging module.
@@ -132,6 +147,12 @@ class BaseLoggerAdapter(logging.LoggerAdapter):
     def dts_ixia_output(self, msg, *args, **kwargs):
         self.log(logging.DTS_IXIA_OUTPUT, msg, *args, **kwargs)
 
+    def dts_virtdut_cmd(self, msg, *args, **kwargs):
+        self.log(logging.DTS_VIRTDUT_CMD, msg, *args, **kwargs)
+
+    def dts_virtdut_output(self, msg, *args, **kwargs):
+        self.log(logging.DTS_VIRTDUT_OUTPUT, msg, *args, **kwargs)
+
 
 class ColorHandler(logging.StreamHandler):
     """
@@ -150,6 +171,8 @@ class ColorHandler(logging.StreamHandler):
         logging.SUITE_TESTER_CMD: '',  # SYSTEM
         logging.DTS_IXIA_CMD: '',  # SYSTEM
         logging.DTS_IXIA_OUTPUT: '',  # SYSTEM
+        logging.DTS_VIRTDUT_CMD: '',  # SYSTEM
+        logging.DTS_VIRTDUT_OUTPUT: '',  # SYSTEM
         logging.WARN: '\033[01;33m',  # BOLD YELLOW
         logging.DTS_DUT_RESULT: '\033[01;34m',  # BOLD BLUE
         logging.DTS_TESTER_RESULT: '\033[01;34m',  # BOLD BLUE
@@ -189,6 +212,8 @@ class DTSLOG(BaseLoggerAdapter):
         self.crb = crb
         super(DTSLOG, self).__init__(self.logger, dict(crb=self.crb))
 
+        self.salt = ''
+
         self.fh = None
         self.ch = None
 
@@ -221,24 +246,28 @@ class DTSLOG(BaseLoggerAdapter):
         """
         DTS warnning level log function.
         """
+        message = add_salt(self.salt, message)
         self.logger.log(self.warn_lvl, message)
 
     def info(self, message):
         """
         DTS information level log function.
         """
+        message = add_salt(self.salt, message)
         self.logger.log(self.info_lvl, message)
 
     def error(self, message):
         """
         DTS error level log function.
         """
+        message = add_salt(self.salt, message)
         self.logger.log(self.error_lvl, message)
 
     def debug(self, message):
         """
         DTS debug level log function.
         """
+        message = add_salt(self.salt, message)
         self.logger.log(self.debug_lvl, message)
 
     def set_logfile_path(self, path):
@@ -270,17 +299,34 @@ class DTSLOG(BaseLoggerAdapter):
         ch = ColorHandler()
         self.__log_hander(fh, ch)
 
-        if crb == "dut":
+        def set_salt(crb, start_flag):
+            if LOG_NAME_SEP in crb:
+                old = '%s%s' % (start_flag, LOG_NAME_SEP)
+                if not self.salt:
+                    self.salt = crb.replace(old, '', 1)
+
+        if crb.startswith('dut'):
             self.info_lvl = logging.DTS_DUT_CMD
             self.debug_lvl = logging.DTS_DUT_OUTPUT
             self.warn_lvl = logging.DTS_DUT_RESULT
-        elif crb == "tester":
+
+            set_salt(crb, 'dut')
+        elif crb.startswith('tester'):
             self.info_lvl = logging.DTS_TESTER_CMD
             self.debug_lvl = logging.DTS_TESTER_OUTPUT
             self.warn_lvl = logging.DTS_TESTER_RESULT
-        elif crb == "ixia":
+
+            set_salt(crb, 'tester')
+        elif crb.startswith('ixia'):
             self.info_lvl = logging.DTS_IXIA_CMD
             self.debug_lvl = logging.DTS_IXIA_OUTPUT
+
+            set_salt(crb, 'ixia')
+        elif crb.startswith('virtdut'):
+            self.info_lvl = logging.DTS_VIRTDUT_CMD
+            self.debug_lvl = logging.DTS_VIRTDUT_OUTPUT
+
+            set_salt(crb, 'virtdut')
         else:
             self.error_lvl = logging.ERROR
             self.warn_lvl = logging.WARNING
@@ -296,15 +342,18 @@ class DTSLOG(BaseLoggerAdapter):
         ch = ColorHandler()
         self.__log_hander(fh, ch)
 
-        if crb == "dut":
+        if crb == 'dut':
             self.info_lvl = logging.SUITE_DUT_CMD
             self.debug_lvl = logging.SUITE_DUT_OUTPUT
-        elif crb == "tester":
+        elif crb == 'tester':
             self.info_lvl = logging.SUITE_TESTER_CMD
             self.debug_lvl = logging.SUITE_TESTER_OUTPUT
-        elif crb == "ixia":
+        elif crb == 'ixia':
             self.info_lvl = logging.DTS_IXIA_CMD
             self.debug_lvl = logging.DTS_IXIA_OUTPUT
+        elif crb == 'virtdut':
+            self.info_lvl = logging.DTS_VIRTDUT_CMD
+            self.debug_lvl = logging.DTS_VIRTDUT_OUTPUT
 
     def logger_exit(self):
         """
diff --git a/framework/main.py b/framework/main.py
index 3e467d0..0496b20 100755
--- a/framework/main.py
+++ b/framework/main.py
@@ -117,6 +117,10 @@ parser.add_argument('-v', '--verbose',
                     action='store_true',
                     help='enable verbose output, all message output on screen')
 
+parser.add_argument('--virttype',
+                    default='kvm',
+                    help='set virt type,support libvirt,xen,kvm')
+
 parser.add_argument('--debug',
                     action='store_true',
                     help='enable debug mode, user can enter debug mode in process')
@@ -136,4 +140,5 @@ if args.git is not None:
 dts.run_all(args.config_file, args.snapshot, args.git,
             args.patch, args.skip_setup, args.read_cache,
             args.project, args.suite_dir, args.test_cases,
-            args.dir, args.output, args.verbose, args.debug)
+            args.dir, args.output, args.verbose,args.virttype,
+            args.debug)
diff --git a/framework/project_dpdk.py b/framework/project_dpdk.py
index 8963924..67bd492 100644
--- a/framework/project_dpdk.py
+++ b/framework/project_dpdk.py
@@ -39,7 +39,7 @@ from crb import Crb
 from dut import Dut
 from tester import Tester
 from logger import getLogger
-from settings import IXIA
+from settings import IXIA, accepted_nic
 
 
 class DPDKdut(Dut):
@@ -50,8 +50,9 @@ class DPDKdut(Dut):
     """
 
     def __init__(self, crb, serializer):
-        self.NAME = 'dut'
+ 
         super(DPDKdut, self).__init__(crb, serializer)
+        self.testpmd = None
 
     def set_target(self, target):
         """
@@ -60,6 +61,7 @@ class DPDKdut(Dut):
         Set hugepage on DUT and install modules required by DPDK.
         Configure default ixgbe PMD function.
         """
+        self.target = target
         self.set_toolchain(target)
 
         # set env variable
@@ -99,7 +101,16 @@ class DPDKdut(Dut):
             out = self.send_expect("lsmod | grep igb_uio", "#")
             if "igb_uio" in out:
                 self.send_expect("rmmod -f igb_uio", "#", 70)
-            self.send_expect("insmod ./" + target + "/kmod/igb_uio.ko", "#", 60)
+            if "rte_dom0_mm" in self.send_expect("lsmod |grep rte_dom0_mm", "#"):
+                self.send_expect("rmmod -f rte_dmo0_mm", "#", 70)
+            self.send_expect(
+                "insmod ./" + target + "/kmod/igb_uio.ko", "#", 60)
+            if self.virttype == 'xen':
+                self.send_expect(
+                    "insmod ./" + target + "/kmod/rte_dom0_mm.ko", "#", 60)
+                self.send_expect(
+                    r'echo 1024 > /sys/kernel/mm/dom0-mm/memsize-mB/memsize', "#", 70)
+
             out = self.send_expect("lsmod | grep igb_uio", "#")
             assert ("igb_uio" in out), "Failed to insmod igb_uio"
 
@@ -110,7 +121,7 @@ class DPDKdut(Dut):
         binding_list = ''
 
         for (pci_bus, pci_id) in self.pci_devices_info:
-            if dts.accepted_nic(pci_id):
+            if accepted_nic(pci_id):
                 binding_list += '%s,' % (pci_bus)
 
         self.send_expect("kldunload if_ixgbe.ko", "#")
@@ -158,6 +169,9 @@ class DPDKdut(Dut):
         """
         # clean all
         self.send_expect("rm -rf " + target, "#")
+        if self.virttype == 'xen':
+            self.send_expect("sed -i -e 's/CONFIG_RTE_LIBRTE_XEN_DOM0=.*$/"
+                             + "CONFIG_RTE_LIBRTE_XEN_DOM0=y/' config/common_linuxapp", "# ", 30)
 
         # compile
         out = self.send_expect("make -j install T=%s %s" % (target, extra_options), "# ", 120)
@@ -178,9 +192,9 @@ class DPDKdut(Dut):
         self.send_expect("rm -rf " + target, "#")
 
         # compile
-        out = self.send_expect("make -j %d install T=%s CC=gcc48" % (self.number_of_cores,
-                                                                     target),
-                               "#", 120)
+        out = self.send_expect(
+            "make -j %d install T=%s CC=gcc48" %
+            (self.number_of_cores, target), "#", 120)
 
         if("Error" in out or "No rule to make" in out):
             self.logger.error("ERROR - try without '-j'")
@@ -191,10 +205,7 @@ class DPDKdut(Dut):
         assert ("Error" not in out), "Compilation error..."
         assert ("No rule to make" not in out), "No rule to make error..."
 
-    def prerequisites(self, pkgName, patch):
-        """
-        Copy DPDK package to DUT and apply patch files.
-        """
+    def prepare_package(self, pkgName, patch):
         if not self.skip_setup:
             assert (os.path.isfile(pkgName) is True), "Invalid package"
 
@@ -202,7 +213,7 @@ class DPDKdut(Dut):
             # ToDo: make this configurable
             dst_dir = "/tmp/"
 
-            out = self.send_expect("ls %s && cd %s" % (dst_dir, p_dir),
+            out = self.send_expect("ll %s && cd %s" % (dst_dir, p_dir),
                                    "#", verify=True)
             if out == -1:
                 raise ValueError("Directiry %s or %s does not exist,"
@@ -249,6 +260,17 @@ class DPDKdut(Dut):
                                            (self.base_dir, dst_dir + p), "# ")
                     assert "****" not in out
 
+            self.session.copy_file_to("dep/aclrule.tgz", dst_dir)
+            # unpack acl rule
+            out = self.send_expect("tar zxf %saclrule.tgz -C %s" % (dst_dir, p_dir), "# ", 20, verify=True)
+            if out == -1:
+                raise ValueError("acl rule extract failure!!!")
+
+    def prerequisites(self, pkgName, patch):
+        """
+        Copy DPDK package to DUT and apply patch files.
+        """
+        self.prepare_package(pkgName, patch)
         self.dut_prerequisites()
 
     def bind_interfaces_linux(self, driver='igb_uio', nics_to_bind=None):
@@ -354,16 +376,20 @@ class DPDKtester(Tester):
             total_huge_pages = self.get_total_huge_pages()
             if total_huge_pages == 0:
                 self.mount_huge_pages()
-                self.set_huge_pages(4096)
+                self.set_huge_pages(1024)
 
             self.session.copy_file_to("dep/tgen.tgz")
             self.session.copy_file_to("dep/tclclient.tgz")
+            self.session.copy_file_to("dep/aclpcap.tgz")
             # unpack tgen
             out = self.send_expect("tar zxf tgen.tgz", "# ")
             assert "Error" not in out
             # unpack tclclient
             out = self.send_expect("tar zxf tclclient.tgz", "# ")
             assert "Error" not in out
+            # unpacl ACL pcap files
+            out = self.send_expect("tar zxf aclpcap.tgz", "# ")
+            assert "Error" not in out
 
         self.send_expect("modprobe uio", "# ")
 
@@ -386,10 +412,10 @@ class DPDKtester(Tester):
         """
         hugepages_size = self.send_expect("awk '/Hugepagesize/ {print $2}' /proc/meminfo", "# ")
 
-        if int(hugepages_size) < (1024 * 1024):
-            arch_huge_pages = hugepages if hugepages > 0 else 4096
+        if int(hugepages_size) < (2048 * 2048):
+            arch_huge_pages = hugepages if hugepages > 0 else 2048
             total_huge_pages = self.get_total_huge_pages()
 
-            self.mount_huge_pages()
-            if total_huge_pages != arch_huge_pages:
-                self.set_huge_pages(arch_huge_pages)
+        self.mount_huge_pages()
+        if total_huge_pages != arch_huge_pages:
+            self.set_huge_pages(arch_huge_pages)
-- 
1.9.0