From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga09.intel.com (mga09.intel.com [134.134.136.24]) by dpdk.org (Postfix) with ESMTP id 53F2EB6AB for ; Mon, 18 May 2015 07:07:39 +0200 (CEST) Received: from orsmga002.jf.intel.com ([10.7.209.21]) by orsmga102.jf.intel.com with ESMTP; 17 May 2015 22:07:38 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.13,450,1427785200"; d="scan'208";a="730938743" Received: from shvmail01.sh.intel.com ([10.239.29.42]) by orsmga002.jf.intel.com with ESMTP; 17 May 2015 22:07:38 -0700 Received: from shecgisg003.sh.intel.com (shecgisg003.sh.intel.com [10.239.29.90]) by shvmail01.sh.intel.com with ESMTP id t4I57Z0p022434; Mon, 18 May 2015 13:07:35 +0800 Received: from shecgisg003.sh.intel.com (localhost [127.0.0.1]) by shecgisg003.sh.intel.com (8.13.6/8.13.6/SuSE Linux 0.8) with ESMTP id t4I57Xex001362; Mon, 18 May 2015 13:07:35 +0800 Received: (from huilongx@localhost) by shecgisg003.sh.intel.com (8.13.6/8.13.6/Submit) id t4I57XQO001358; Mon, 18 May 2015 13:07:33 +0800 From: sjiajiax To: dts@dpdk.org Date: Mon, 18 May 2015 13:07:19 +0800 Message-Id: <1431925646-1314-3-git-send-email-sunx.jiajia@intel.com> X-Mailer: git-send-email 1.7.4.1 In-Reply-To: <1431925646-1314-1-git-send-email-sunx.jiajia@intel.com> References: <1431925646-1314-1-git-send-email-sunx.jiajia@intel.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Subject: [dts] =?utf-8?b?W+KAmGR0cy12MeKAmSAyLzldIE9wdGltaXplIHNzaCBjb25u?= =?utf-8?q?ection?= X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 18 May 2015 05:07:40 -0000 Optimize configure parse Move some functions in dts to an common utils module Signed-off-by: sjiajiax --- framework/config.py | 171 +++++++++++++++++++++++++++++++++++--------- framework/ssh_connection.py | 5 ++ framework/ssh_pexpect.py | 63 +++++++++++++--- framework/utils.py | 77 ++++++++++++++++++++ 4 files changed, 270 insertions(+), 46 deletions(-) mode change 100755 => 100644 framework/config.py create mode 100644 framework/utils.py diff --git a/framework/config.py b/framework/config.py old mode 100755 new mode 100644 index d2548e8..927c846 --- a/framework/config.py +++ b/framework/config.py @@ -37,45 +37,133 @@ import re import ConfigParser # config parse module import argparse # prase arguments module -portconf = "conf/ports.cfg" -crbconf = "conf/crbs.cfg" +PORTCONF = "conf/ports.cfg" +CRBCONF = "conf/crbs.cfg" +VIRTCONF = "conf/virt_global.cfg" class UserConf(): - def __init__(self, port_conf=portconf, crb_conf=crbconf): - self.port_config = port_conf - self.crb_config = crb_conf + def __init__(self, config): + self.conf = ConfigParser.SafeConfigParser() + load_files = self.conf.read(config) + if load_files == []: + print "FAILED LOADING %s!!!" % config + self.conf = None + raise + + def get_sections(self): + if self.conf is None: + return None + + return self.conf.sections() + + def load_section(self, section): + if self.conf is None: + return None + + items = None + for conf_sect in self.conf.sections(): + if conf_sect == section: + items = self.conf.items(section) + + return items + + def load_config(self, item): + confs = [conf.strip() for conf in item.split(';')] + if '' in confs: + confs.remove('') + return confs + + def load_param(self, conf): + paramDict = dict() + + for param in conf.split(','): + (key, _, value) = param.partition('=') + paramDict[key] = value + return paramDict + + +class VirtConf(UserConf): + + def __init__(self, virt_conf=VIRTCONF): + self.config_file = virt_conf + self.virt_cfg = {} + try: + self.virt_conf = UserConf(self.config_file) + except Exception as e: + print "FAILED LOADING VIRT CONFIG!!!" + self.virt_conf = None + + def load_virt_config(self, name): + self.virt_cfgs = [] + + try: + virt_confs = self.virt_conf.load_section(name) + except: + print "FAILED FIND SECTION %s!!!" % name + return + + for virt_conf in virt_confs: + virt_cfg = {} + virt_params = [] + key, config = virt_conf + confs = self.virt_conf.load_config(config) + for config in confs: + virt_params.append(self.load_virt_param(config)) + virt_cfg[key] = virt_params + self.virt_cfgs.append(virt_cfg) + + def get_virt_config(self): + return self.virt_cfgs + + def load_virt_param(self, config): + cfg_params = self.virt_conf.load_param(config) + return cfg_params + + +class PortConf(UserConf): + + def __init__(self, port_conf=PORTCONF): + self.config_file = port_conf self.ports_cfg = {} self.pci_regex = "([\da-f]{2}:[\da-f]{2}.\d{1})$" try: - self.port_conf = ConfigParser.SafeConfigParser() - self.port_conf.read(self.port_config) + self.port_conf = UserConf(self.config_file) except Exception as e: print "FAILED LOADING PORT CONFIG!!!" + self.port_conf = None def load_ports_config(self, crbIP): - ports = [] - for crb in self.port_conf.sections(): - if crb != crbIP: - continue - ports = [port.strip() - for port in self.port_conf.get(crb, 'ports').split(';')] + self.ports_cfg = {} + if self.port_conf is None: + return + + ports = self.port_conf.load_section(crbIP) + if ports is None: + return + key, config = ports[0] + confs = self.port_conf.load_config(config) + + for config in confs: + port_param = self.port_conf.load_param(config) - for port in ports: - port_cfg = self.__parse_port_param(port) # check pci BDF validity - if 'pci' not in port_cfg: + if 'pci' not in port_param: print "NOT FOUND CONFIG FOR NO PCI ADDRESS!!!" continue - m = re.match(self.pci_regex, port_cfg['pci']) + m = re.match(self.pci_regex, port_param['pci']) if m is None: print "INVALID CONFIG FOR NO PCI ADDRESS!!!" continue - keys = port_cfg.keys() + keys = port_param.keys() keys.remove('pci') - self.ports_cfg[port_cfg['pci']] = {key: port_cfg[key] for key in keys} + self.ports_cfg[port_param['pci']] = { + key: port_param[key] for key in keys} + if 'numa' in self.ports_cfg[port_param['pci']]: + numa_str = self.ports_cfg[port_param['pci']]['numa'] + self.ports_cfg[port_param['pci']]['numa'] = int(numa_str) def get_ports_config(self): return self.ports_cfg @@ -86,23 +174,36 @@ class UserConf(): else: return False - def __parse_port_param(self, port): - portDict = dict() - - for param in port.split(','): - (key, _, value) = param.partition('=') - if key == 'numa': - portDict[key] = int(value) - else: - portDict[key] = value - return portDict + if __name__ == '__main__': - parser = argparse.ArgumentParser(description="Load DTS configuration files") - parser.add_argument("-p", "--portconf", default=portconf) - parser.add_argument("-c", "--crbconf", default=crbconf) + parser = argparse.ArgumentParser( + description="Load DTS configuration files") + parser.add_argument("-p", "--portconf", default=PORTCONF) + parser.add_argument("-c", "--crbconf", default=CRBCONF) + parser.add_argument("-v", "--virtconf", default=VIRTCONF) args = parser.parse_args() - conf = UserConf() - conf.load_ports_config('192.168.1.1') - conf.check_port_available('0000:86:00.0') + + # not existed configuration file + VirtConf('/tmp/not-existed.cfg') + + # example for basic use configuration file + conf = UserConf(PORTCONF) + for section in conf.get_sections(): + items = conf.load_section(section) + key, value = items[0] + confs = conf.load_config(value) + for config in confs: + conf.load_param(config) + + # example for port configuration file + portconf = PortConf(PORTCONF) + portconf.load_ports_config('DUT IP') + print portconf.get_ports_config() + portconf.check_port_available('86:00.0') + + # example for global virtualization configuration file + virtconf = VirtConf(VIRTCONF) + virtconf.load_virt_config('LIBVIRT') + print virtconf.get_virt_config() diff --git a/framework/ssh_connection.py b/framework/ssh_connection.py index 18a6517..7286b14 100644 --- a/framework/ssh_connection.py +++ b/framework/ssh_connection.py @@ -62,6 +62,11 @@ class SSHConnection(object): self.logger.debug(out) return out + def get_session_before(self, timeout=15): + out = self.session.get_session_before(timeout) + self.logger.debug(out) + return out + def close(self): self.session.close() diff --git a/framework/ssh_pexpect.py b/framework/ssh_pexpect.py index 735df44..4193020 100644 --- a/framework/ssh_pexpect.py +++ b/framework/ssh_pexpect.py @@ -2,7 +2,8 @@ import time import pexpect import pxssh from debugger import ignore_keyintr, aware_keyintr -from exception import TimeoutException, SSHConnectionException +from exception import TimeoutException, SSHConnectionException, SSHSessionDeadException +from utils import RED, GREEN """ Module handle ssh sessions between tester and DUT. @@ -14,16 +15,28 @@ Aslo support transfer files to tester or DUT. class SSHPexpect(object): def __init__(self, host, username, password): - self.magic_prompt = "[MAGIC PROMPT]" + self.magic_prompt = "MAGIC PROMPT" try: self.session = pxssh.pxssh() - self.username = username self.host = host + self.username = username self.password = password - self.session.login(self.host, self.username, - self.password, original_prompt='[$#>]') + if ':' in host: + self.ip = host.split(':')[0] + self.port = int(host.split(':')[1]) + self.session.login(self.ip, self.username, + self.password, original_prompt='[$#>]', + port=self.port, login_timeout=20) + else: + self.session.login(self.host, self.username, + self.password, original_prompt='[$#>]') self.send_expect('stty -echo', '# ', timeout=2) - except Exception: + except Exception, e: + print RED(e) + if getattr(self, 'port', None): + suggestion = "\nSuggession: Check if the fireware on [ %s ] " % \ + self.ip + "is stoped\n" + print GREEN(suggestion) raise SSHConnectionException(host) def init_log(self, logger, name): @@ -33,7 +46,7 @@ class SSHPexpect(object): def send_expect_base(self, command, expected, timeout=15): ignore_keyintr() - self.__flush() # clear buffer + self.__flush() # clear buffer self.session.PROMPT = expected self.__sendline(command) self.__prompt(command, timeout) @@ -45,7 +58,7 @@ class SSHPexpect(object): def send_expect(self, command, expected, timeout=15, verify=False): ret = self.send_expect_base(command, expected, timeout) if verify: - ret_status = self.send_expect_base("echo $?", expected) + ret_status = self.send_expect_base("echo $?", expected, timeout) if not int(ret_status): return ret else: @@ -54,21 +67,44 @@ class SSHPexpect(object): else: return ret - def __flush(self): + def get_session_before(self, timeout=15): + """ + Get all output before timeout + """ + ignore_keyintr() self.session.PROMPT = self.magic_prompt - self.session.prompt(0.1) + try: + self.session.prompt(timeout) + except Exception as e: + pass + + aware_keyintr() + before = self.get_output_before() + self.__flush() + return before + + def __flush(self): + """ + Clear all session buffer + """ + self.session.buffer = "" + self.session.before = "" def __prompt(self, command, timeout): if not self.session.prompt(timeout): raise TimeoutException(command, self.get_output_all()) def __sendline(self, command): + if not self.isalive(): + raise SSHSessionDeadException(self.host) if len(command) == 2 and command.startswith('^'): self.session.sendcontrol(command[1]) else: self.session.sendline(command) def get_output_before(self): + if not self.isalive(): + raise SSHSessionDeadException(self.host) self.session.flush() before = self.session.before.rsplit('\r\n', 1) if before[0] == "[PEXPECT]": @@ -103,7 +139,12 @@ class SSHPexpect(object): """ Sends a local file to a remote place. """ - command = 'scp {0} {1}@{2}:{3}'.format(src, self.username, self.host, dst) + if ':' in self.host: + command = 'scp -P {0} -o NoHostAuthenticationForLocalhost=yes {1} {2}@{3}:{4}'.format( + str(self.port), src, self.username, self.ip, dst) + else: + command = 'scp {0} {1}@{2}:{3}'.format( + src, self.username, self.host, dst) if password == '': self._spawn_scp(command, self.password) else: diff --git a/framework/utils.py b/framework/utils.py new file mode 100644 index 0000000..57eb988 --- /dev/null +++ b/framework/utils.py @@ -0,0 +1,77 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json # json format +import re + + +def RED(text): + return "\x1B[" + "31;1m" + str(text) + "\x1B[" + "0m" + + +def BLUE(text): + return "\x1B[" + "36;1m" + str(text) + "\x1B[" + "0m" + + +def GREEN(text): + return "\x1B[" + "32;1m" + str(text) + "\x1B[" + "0m" + + +def pprint(some_dict): + """ + Print JSON format dictionary object. + """ + return json.dumps(some_dict, sort_keys=True, indent=4) + + +def regexp(s, to_match, allString=False): + """ + Ensure that the re `to_match' only has one group in it. + """ + + scanner = re.compile(to_match, re.DOTALL) + if allString: + return scanner.findall(s) + m = scanner.search(s) + if m is None: + print RED("Failed to match " + to_match + " in the string " + s) + return None + return m.group(1) + + +def get_obj_funcs(obj, func_name_regex): + """ + Return function list which name matched regex. + """ + for func_name in dir(obj): + func = getattr(obj, func_name) + if callable(func) and re.match(func_name_regex, func.__name__): + yield func -- 1.9.0