From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by dpdk.space (Postfix) with ESMTP id D9213A046B for ; Mon, 24 Jun 2019 12:45:56 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id D1A062C2B; Mon, 24 Jun 2019 12:45:56 +0200 (CEST) Received: from mail-pl1-f175.google.com (mail-pl1-f175.google.com [209.85.214.175]) by dpdk.org (Postfix) with ESMTP id 7A8FF2C2B for ; Mon, 24 Jun 2019 12:45:55 +0200 (CEST) Received: by mail-pl1-f175.google.com with SMTP id m7so6636807pls.8 for ; Mon, 24 Jun 2019 03:45:55 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=from:to:subject:date:message-id; bh=cocnIIoRNnT7SS9AHHJoeXcPXiYt7e2PBEn54nCQzhw=; b=qSKx9tcqucGuBzNlzGyyACZ2xl1jVMY71AZZ3uXI0Qdi4OfY4TwiFmAa3mvmTKv7lH dDqyqf8MeL3C1GVyHsXKEcxMWn26krR5rZBt2+RoiFQEy2aTfrLRCZ/npDyBg5EFsY5E J306GhZ+1+Cma3C3LmSMFCpHLLYsb5+sQuAyq/ShrlgzqxfG1FbfPazo10MWRvmBt+nc jVSMEx4ZA3ftNFmVbNZeZ+YKDC//tiGYuFpRNZIh+N/lN3d8dqbdcLfdxk5mjXHN5bUk JvHUz5dyoO8VVI0SsTf64JX3P/3D5RstX8+atnEqkfoGBzxuFp9Y0y2BTbhy1QbcJ970 NeAA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:subject:date:message-id; bh=cocnIIoRNnT7SS9AHHJoeXcPXiYt7e2PBEn54nCQzhw=; b=mwg5iHmI5mNL24OrP427WIOY0g66yRXgmantWS/SCXDXPl6V3Oqn5ohJ5Ek3UfLATx NnQW/+qfjZoevEjgQ29yKgl8ITax0zXOUe5rCfoJTnEntyPRowRk4vKl5oBX+zAEqwl1 3TzGxWUK+OHf7ozwOXFQKTz/fNHEHhVW/MfxmXMWFISH+qo+nbff1IY/WHiXHA6ObOYz OdGj0X5uOHGXcBa3eAij4LiYyMqgMb9bZGqjqf1cnjiGGLxkNcaiA7VxXXrAmF+qw/V9 w5neTSB3u1J1WmuAYOgy96RwfwqhaxMoO3gZZ2IbFDZy6GkClLrLO9QAwopDSsiLyVKY /fag== X-Gm-Message-State: APjAAAU1IALhv5OM/yNXUQFMAHHGJlJVogvcsE8qqc1BI+jxuXiYv2ww jzWm1r9hB19PGSgFQbJgMht/DFGk X-Google-Smtp-Source: APXvYqwg53h/te+F2imr8xWuOc8lVXniCjkV0un3UvVyga89K271b0Smn3od9cZ/f/HZJDNmBHZ4QQ== X-Received: by 2002:a17:902:7297:: with SMTP id d23mr134370364pll.254.1561373154571; Mon, 24 Jun 2019 03:45:54 -0700 (PDT) Received: from localhost.localdomain ([192.47.164.146]) by smtp.gmail.com with ESMTPSA id i17sm7345237pgn.71.2019.06.24.03.45.53 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Mon, 24 Jun 2019 03:45:54 -0700 (PDT) From: yasufum.o@gmail.com To: spp@dpdk.org, ferruh.yigit@intel.com, yasufum.o@gmail.com Date: Mon, 24 Jun 2019 19:45:50 +0900 Message-Id: <20190624104550.25253-1-yasufum.o@gmail.com> X-Mailer: git-send-email 2.17.1 Subject: [spp] [PATCH] controller: suggest empty lcores for launch cmd X-BeenThere: spp@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: Soft Patch Panel List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: spp-bounces@dpdk.org Sender: "spp" From: Yasufumi Ogawa This update is for completion of `pri; launch` command to suggest slave lcores while checking lcore usage. Master lcore and number of slave lcores are defined as config params, and each of slave lcores are selected from empty lcores. CPU layout and usage are retrieved via spp-ctl REST APIs. This update does not support a case if there are not enough number of empty lcores yet, for which SPP CLI should show warning or interrupt to run the command. Signed-off-by: Yasufumi Ogawa --- src/controller/commands/pri.py | 142 ++++++++++++++++++++++++------ src/controller/config/default.yml | 4 +- 2 files changed, 117 insertions(+), 29 deletions(-) diff --git a/src/controller/commands/pri.py b/src/controller/commands/pri.py index 97f8780..654b7dd 100644 --- a/src/controller/commands/pri.py +++ b/src/controller/commands/pri.py @@ -27,7 +27,7 @@ class SppPrimary(object): # TODO(yasufum) replace placeholders __XXX__ to {keyword}. # Setup template of args for `pri; launch` - temp = "-l __BASE_LCORE__,{wlcores} " + temp = "-l __MASTER_LCORE__,{slcores} " temp = temp + "__MEM__ " temp = temp + "-- " temp = temp + "{opt_sid} {sid} " # '-n 1' or '--client-id 1' @@ -155,6 +155,79 @@ class SppPrimary(object): rid=rports['id'], rx=rports['rx'], tx=rports['tx'], rx_drop=rports['rx_drop'], tx_drop=rports['tx_drop'])) + def _get_empty_lcores(self): + """Get lcore usage from spp-ctl for making launch options. + + Return value is a double dimension list of unsed lcores. + [[2,3,...], [16,17,...]] + + To get the result, get CPU layout as an list first, then remove + used lcores from the list. + """ + + sockets = [] # A set of CPU sockets. + # Get list of CPU layout + res = self.spp_ctl_cli.get('cpu_layout') + if res is not None: + if res.status_code == 200: + try: + # Get layout of each of sockets as an array. + # [[0,1,2,3,..., 15], [16,17,18],...]] + socket = [] + for sock in res.json(): + for ent in sock['cores']: + socket.append(ent['lcores']) + socket.sort() + socket = sum(socket, []) + sockets.append(socket) + + except KeyError as e: + print('Error: {} is not defined!'.format(e)) + + # Get list of used lcores. + res = self.spp_ctl_cli.get('cpu_usage') + if res is not None: + if res.status_code == 200: + try: + p_master_lcore = None + s_master_lcore = None + p_slave_lcores = [] + s_slave_lcores = [] + + cpu_usage = res.json() + for ent in cpu_usage: + if ent['proc-type'] == 'primary': + p_master_lcore = ent['master-lcore'] + p_slave_lcores.append(ent['lcores']) + else: + s_master_lcore = ent['master-lcore'] + s_slave_lcores.append(ent['lcores']) + + # Remove duplicated lcores and make them unique. + # sum() is used for flattening two dimension list. + p_slave_lcores = list(set(sum(p_slave_lcores, []))) + s_slave_lcores = list(set(sum(s_slave_lcores, []))) + + # Remove used lcores from `sockets`. + for socket in sockets: + if p_master_lcore is not None: + if p_master_lcore in socket: + socket.remove(p_master_lcore) + if s_master_lcore is not None: + if s_master_lcore in socket: + socket.remove(s_master_lcore) + for l in p_slave_lcores: + if l in socket: + socket.remove(l) + for l in s_slave_lcores: + if l in socket: + socket.remove(l) + + except KeyError as e: + print('Error: {} is not defined!'.format(e)) + + return sockets + def _setup_launch_opts(self, tokens, cli_config): """Make options for launch cmd from config params.""" @@ -164,8 +237,13 @@ class SppPrimary(object): if (tokens[2] in spp_common.SEC_TYPES) and \ (int(tokens[3])-1 in range(max_secondary)): ptype = tokens[2] + + # TODO(yasufum) Not accept if given sec ID is already + # used. sid = tokens[3] + # Option of secondary ID is different between spp_nfv + # and others. if ptype == 'nfv': opt_sid = '-n' else: @@ -176,34 +254,27 @@ class SppPrimary(object): server_addr = common.current_server_addr() server_addr = server_addr.replace('7777', '6666') - # Lcore ID of worker lcore starts from sec ID in - # default. - lcore_base = int(sid) - - # Define rest of worker lcores from config dynamically. - if ptype == 'nfv': # one worker lcore is enough + # Define number of slave lcores and other options + # from config. + if ptype == 'nfv': # one slave lcore is enough if 'sec_nfv_nof_lcores' in cli_config.keys(): tmpkey = 'sec_nfv_nof_lcores' - nof_workers = int( + nof_slaves = int( cli_config[tmpkey]['val']) - elif ptype == 'vf': if 'sec_vf_nof_lcores' in cli_config.keys(): - nof_workers = int( + nof_slaves = int( cli_config['sec_vf_nof_lcores']['val']) - - elif ptype == 'mirror': # two worker cores + elif ptype == 'mirror': # two slave lcores if 'sec_mirror_nof_lcores' in cli_config.keys(): tmpkey = 'sec_mirror_nof_lcores' - nof_workers = int( + nof_slaves = int( cli_config[tmpkey]['val']) - - elif ptype == 'pcap': # at least two worker cores + elif ptype == 'pcap': # at least two slave lcores if 'sec_pcap_nof_lcores' in cli_config.keys(): tmpkey = 'sec_pcap_nof_lcores' - nof_workers = int( + nof_slaves = int( cli_config[tmpkey]['val']) - if 'sec_pcap_port' in cli_config.keys(): temp = '-c {}'.format( cli_config['sec_pcap_port']['val']) @@ -211,20 +282,37 @@ class SppPrimary(object): self.launch_template = '{} {}'.format( self.launch_template, temp) - last_core = lcore_base + nof_workers - 1 + # Get and flatten empty lcores on each of sockets. + empty_lcores = self._get_empty_lcores() + empty_lcores = sum(empty_lcores, []) + + if 'sec_m_lcore' in cli_config.keys(): + m_lcore_id = int(cli_config['sec_m_lcore']['val']) # Decide lcore option based on configured number of # lcores. - if last_core == lcore_base: - rest_core = '{}'.format(last_core) - else: - rest_core = '{}-{}'.format(lcore_base, last_core) - + slave_lcores = [] + for l in empty_lcores: + # Master lcore ID should be smaller than slaves. + if l > m_lcore_id: + slave_lcores.append(str(l)) + # TODO(yasufum) warn if enough number of empty + # lcores cannot be assinged. + if len(slave_lcores) > (nof_slaves - 1): + break + + # TODO(yasufum) make smarter form, for example, + # change '1,2,3' to '1-3'. + slave_lcores = ','.join(slave_lcores) + + # Replace labels in template with actual params to make + # candidate options. temp = self._setup_launch_template( cli_config, self.launch_template) candidates = [temp.format( - wlcores=rest_core, opt_sid=opt_sid, sid=sid, + slcores=slave_lcores, opt_sid=opt_sid, sid=sid, sec_addr=server_addr)] + else: logger.error( 'Error: max_secondary is not defined in config') @@ -287,9 +375,9 @@ class SppPrimary(object): sec_mem = cli_config['sec_mem']['val'] template = template.replace('__MEM__', sec_mem) - if 'sec_base_lcore' in cli_config.keys(): - sec_base_lcore = cli_config['sec_base_lcore']['val'] - template = template.replace('__BASE_LCORE__', str(sec_base_lcore)) + if 'sec_m_lcore' in cli_config.keys(): + sec_m_lcore = cli_config['sec_m_lcore']['val'] + template = template.replace('__MASTER_LCORE__', str(sec_m_lcore)) if 'sec_vhost_cli' in cli_config.keys(): if cli_config['sec_vhost_cli']['val']: diff --git a/src/controller/config/default.yml b/src/controller/config/default.yml index 7625451..a746c9a 100644 --- a/src/controller/config/default.yml +++ b/src/controller/config/default.yml @@ -13,9 +13,9 @@ topo_size: sec_mem: val: -m 512 desc: Mem size -sec_base_lcore: +sec_m_lcore: val: 1 - desc: Shared lcore among secondaries + desc: Shared master lcore among secondaries sec_wait_launch: val: 0.5 desc: Wait for launching secondary process in sec -- 2.17.1