From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 712AB429E4; Tue, 25 Apr 2023 05:05:53 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 6AA3B40ED4; Tue, 25 Apr 2023 05:05:53 +0200 (CEST) Received: from mga17.intel.com (mga17.intel.com [192.55.52.151]) by mails.dpdk.org (Postfix) with ESMTP id 2B9E8400D7 for ; Tue, 25 Apr 2023 05:05:50 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1682391951; x=1713927951; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=JGlJDXtTK3ku78iRBk+frOHSOB7uOludKMQtuqSoQLI=; b=PY7mO/gtQ5Ca1uDmKgk7bc+8jPrLzdp+MhXofNPBm5T10L8vxxvq/c2g Nv3owX7WuICH5nIz6IoVg2ros6Oq/bSTPc96Q0ANmVCbWNqPoQjT55K9h oFX+GnmfDhrVCGFesiPIIWDHGs2y6wVJxETyh+9pfEipXEiqMwrssvRAm TPYjyiyYN8qcMbsKrbFPrySLO9B2S3QhImlDrkbf5hHOhXrhZ6IzzYTzJ q7e4ZDy9TKRKF84xOA1RJZFKjr9NMUAuc8Or5lOxQllX7uGekG05+ZoeK e5WsFKa0jUDLDzwGiTdE3Gzd2DRJoIZcvsuzDPuAzoAA73wJgg2oMx5FM A==; X-IronPort-AV: E=McAfee;i="6600,9927,10690"; a="326945825" X-IronPort-AV: E=Sophos;i="5.99,224,1677571200"; d="scan'208";a="326945825" Received: from fmsmga006.fm.intel.com ([10.253.24.20]) by fmsmga107.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Apr 2023 20:05:50 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6600,9927,10690"; a="939584795" X-IronPort-AV: E=Sophos;i="5.99,224,1677571200"; d="scan'208";a="939584795" Received: from unknown (HELO localhost.localdomain) ([10.239.252.222]) by fmsmga006-auth.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Apr 2023 20:05:48 -0700 From: Wei Ling To: dts@dpdk.org Cc: Wei Ling Subject: [dts][PATCH V1 1/2] tests/pvp_virtio_user_2M_hugepages: add expected value in suite config to verify Date: Tue, 25 Apr 2023 11:03:43 +0800 Message-Id: <20230425030344.2958301-2-weix.ling@intel.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20230425030344.2958301-1-weix.ling@intel.com> References: <20230425030344.2958301-1-weix.ling@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dts-bounces@dpdk.org Optimize the verify expected value from hard code to read from conf/pvp_virtio_user_2M_hugepages.cfg to adapt different platform. Signed-off-by: Wei Ling --- .../TestSuite_pvp_virtio_user_2M_hugepages.py | 141 +++++++++++------- 1 file changed, 90 insertions(+), 51 deletions(-) diff --git a/tests/TestSuite_pvp_virtio_user_2M_hugepages.py b/tests/TestSuite_pvp_virtio_user_2M_hugepages.py index 7fe0bcbb..e48e772a 100644 --- a/tests/TestSuite_pvp_virtio_user_2M_hugepages.py +++ b/tests/TestSuite_pvp_virtio_user_2M_hugepages.py @@ -2,20 +2,14 @@ # Copyright(c) 2019 Intel Corporation # -""" -DPDK Test suite. -vhost/virtio-user pvp with 2M hugepage. -""" +from copy import deepcopy -import time - -import framework.utils as utils from framework.pktgen import PacketGeneratorHelper -from framework.settings import HEADER_SIZE +from framework.settings import HEADER_SIZE, UPDATE_EXPECTED, load_global_setting from framework.test_case import TestCase -class TestPVPVirtioWith2Mhuge(TestCase): +class TestPVPVirtioWith2MHugepages(TestCase): def set_up_all(self): """ Run at the start of each test suite. @@ -40,6 +34,8 @@ class TestPVPVirtioWith2Mhuge(TestCase): self.core_list_virtio_user = self.core_list[0:2] self.core_list_vhost_user = self.core_list[2:4] + self.nb_ports = 1 + self.gap = self.get_suite_cfg()["accepted_tolerance"] self.dst_mac = self.dut.get_mac_address(self.dut_ports[0]) self.header_size = HEADER_SIZE["eth"] + HEADER_SIZE["ip"] + HEADER_SIZE["tcp"] self.frame_sizes = [64, 128, 256, 512, 1024, 1518] @@ -59,6 +55,8 @@ class TestPVPVirtioWith2Mhuge(TestCase): self.number_of_ports = 1 self.path = self.dut.apps_name["test-pmd"] self.testpmd_name = self.path.split("/")[-1] + self.vhost_user = self.dut.new_session(suite="vhost-user") + self.virtio_user = self.dut.new_session(suite="virtio-user") def set_up(self): """ @@ -66,37 +64,19 @@ class TestPVPVirtioWith2Mhuge(TestCase): """ self.dut.send_expect("rm -rf ./vhost-net*", "# ") self.dut.send_expect("killall -s INT %s" % self.testpmd_name, "#") - self.vhost_user = self.dut.new_session(suite="vhost-user") - self.virtio_user = self.dut.new_session(suite="virtio-user") + self.throughput = dict() + self.test_result = dict() # Prepare the result table self.table_header = ["Frame"] - self.table_header.append("Mode") self.table_header.append("Mpps") - self.table_header.append("Queue Num") self.table_header.append("% linerate") - self.result_table_create(self.table_header) - @property - def check_value(self): - check_dict = dict.fromkeys(self.frame_sizes) - linerate = { - 64: 0.085, - 128: 0.12, - 256: 0.20, - 512: 0.35, - 1024: 0.50, - 1280: 0.55, - 1518: 0.60, - } - for size in self.frame_sizes: - speed = self.wirespeed(self.nic, size, self.number_of_ports) - check_dict[size] = round(speed * linerate[size], 2) - return check_dict - - def send_and_verify(self): - """ - Send packet with packet generator and verify + def perf_test(self): + """ + Send packet with packet generator """ + self.result_table_create(self.table_header) + self.throughput = {} for frame_size in self.frame_sizes: payload_size = frame_size - self.header_size tgen_input = [] @@ -113,21 +93,73 @@ class TestPVPVirtioWith2Mhuge(TestCase): streams = self.pktgen_helper.prepare_stream_from_tginput( tgen_input, 100, None, self.tester.pktgen ) - _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams) + # set traffic option + traffic_opt = { + "delay": 5, + "duration": self.get_suite_cfg()["test_duration"], + } + _, pps = self.tester.pktgen.measure_throughput( + stream_ids=streams, options=traffic_opt + ) Mpps = pps / 1000000.0 - self.verify( - Mpps > self.check_value[frame_size], - "%s of frame size %d speed verify failed, expect %s, result %s" - % (self.running_case, frame_size, self.check_value[frame_size], Mpps), + line_rate = ( + Mpps * 100 / float(self.wirespeed(self.nic, frame_size, self.nb_ports)) ) - throughput = Mpps * 100 / float(self.wirespeed(self.nic, 64, 1)) - + self.throughput[frame_size] = Mpps results_row = [frame_size] - results_row.append("2M Hugepages") results_row.append(Mpps) - results_row.append("1") - results_row.append(throughput) + results_row.append(line_rate) self.result_table_add(results_row) + self.result_table_print() + + def handle_expected(self): + """ + Update expected numbers to configurate file: $DTS_CFG_FOLDER/$suite_name.cfg + """ + if load_global_setting(UPDATE_EXPECTED) == "yes": + for frame_size in self.frame_sizes: + self.expected_throughput[frame_size] = round( + self.throughput[frame_size], 3 + ) + + def handle_results(self): + """ + results handled process: + 1, save to self.test_results + 2, create test results table + """ + # save test results to self.test_result + header = self.table_header + header.append("Expected Throughput(Mpps)") + header.append("Status") + self.result_table_create(self.table_header) + for frame_size in self.frame_sizes: + wirespeed = self.wirespeed(self.nic, frame_size, self.nb_ports) + ret_data = {} + ret_data[header[0]] = str(frame_size) + _real = float(self.throughput[frame_size]) + _exp = float(self.expected_throughput[frame_size]) + ret_data[header[1]] = "{:.3f}".format(_real) + ret_data[header[2]] = "{:.3f}%".format(_real * 100 / wirespeed) + ret_data[header[3]] = "{:.3f}".format(_exp) + gap = _exp * -self.gap * 0.01 + if _real > _exp + gap: + ret_data[header[4]] = "PASS" + else: + ret_data[header[4]] = "FAIL" + self.test_result[frame_size] = deepcopy(ret_data) + + for frame_size in self.test_result.keys(): + table_row = list() + for i in range(len(header)): + table_row.append(self.test_result[frame_size][header[i]]) + self.result_table_add(table_row) + # present test results to screen + self.result_table_print() + self.verify( + "FAIL" not in self.test_result, + "Excessive gap between test results and expectations", + ) def start_testpmd_as_vhost(self): """ @@ -170,27 +202,33 @@ class TestPVPVirtioWith2Mhuge(TestCase): """ self.virtio_user.send_expect("quit", "# ", 60) self.vhost_user.send_expect("quit", "# ", 60) - self.dut.close_session(self.vhost_user) - self.dut.close_session(self.virtio_user) def test_perf_pvp_virtio_user_split_ring_2M_hugepages(self): """ Basic test for virtio-user 2M hugepage """ + self.expected_throughput = self.get_suite_cfg()["expected_throughput"][ + self.running_case + ] self.start_testpmd_as_vhost() self.start_testpmd_as_virtio() - self.send_and_verify() - self.result_table_print() + self.perf_test() + self.handle_expected() + self.handle_results() self.close_all_apps() def test_perf_pvp_virtio_user_packed_ring_2M_hugepages(self): """ Basic test for virtio-user 2M hugepage """ + self.expected_throughput = self.get_suite_cfg()["expected_throughput"][ + self.running_case + ] self.start_testpmd_as_vhost() self.start_testpmd_as_virtio(packed=True) - self.send_and_verify() - self.result_table_print() + self.perf_test() + self.handle_expected() + self.handle_results() self.close_all_apps() def tear_down(self): @@ -203,4 +241,5 @@ class TestPVPVirtioWith2Mhuge(TestCase): """ Run after each test suite. """ - pass + self.dut.close_session(self.vhost_user) + self.dut.close_session(self.virtio_user) -- 2.25.1