From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id AEC04429D8; Mon, 24 Apr 2023 12:20:08 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id A9ABC410EC; Mon, 24 Apr 2023 12:20:08 +0200 (CEST) Received: from mga07.intel.com (mga07.intel.com [134.134.136.100]) by mails.dpdk.org (Postfix) with ESMTP id 9538F410EC for ; Mon, 24 Apr 2023 12:20:07 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1682331607; x=1713867607; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=0zB8TvuOhv+RRFRhSi38t3Vhc5Q1g00xIeJJ9A4rnuo=; b=Y3HGiFUHuPW0sfCh8wV+xIiq7YIEB8WLqCBI5RUnLhq13mvOy+8TKB+J FvvtdvVFLbNOG1iJ78nqjtxR/JNghfDhkZiDU+q2tfwlsDWrXLrKulKhz P38l/5gUstqEUkBlkvB/+yN6dofHo3fGoopeiXFP7ViJu5tUYWTy9atei s1DrEU2c0o9KaDAJlPh6eDUFR3iz/fxo5pbOU0i4XWV/99/QAiEh8/fVz gjGwJZHK8beMV24Hl//UjtSupgXw8azQUpXI5XKVvtVoL0HT0XYkMkR/d pSE4D8Nf1gwQBGngSLzh0Lu+pryf8gAzwZQC+v5rDOGXRBX1w2N1P/W1j A==; X-IronPort-AV: E=McAfee;i="6600,9927,10689"; a="411688788" X-IronPort-AV: E=Sophos;i="5.99,222,1677571200"; d="scan'208";a="411688788" Received: from orsmga006.jf.intel.com ([10.7.209.51]) by orsmga105.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Apr 2023 03:20:07 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6600,9927,10689"; a="670457636" X-IronPort-AV: E=Sophos;i="5.99,222,1677571200"; d="scan'208";a="670457636" Received: from unknown (HELO localhost.localdomain) ([10.239.252.222]) by orsmga006-auth.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Apr 2023 03:20:06 -0700 From: Wei Ling To: dts@dpdk.org Cc: Wei Ling Subject: [dts][PATCH V1 1/2] tests/pvp_virtio_user_4k_pages: add expected value to verify Date: Mon, 24 Apr 2023 18:18:19 +0800 Message-Id: <20230424101820.2598235-2-weix.ling@intel.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20230424101820.2598235-1-weix.ling@intel.com> References: <20230424101820.2598235-1-weix.ling@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dts-bounces@dpdk.org Optimize the verify the expected value from the hard code to read from conf/pvp_virtio_user_4k_pages.cfg to adapt different platform. Signed-off-by: Wei Ling --- tests/TestSuite_pvp_virtio_user_4k_pages.py | 131 +++++++++++++------- 1 file changed, 86 insertions(+), 45 deletions(-) diff --git a/tests/TestSuite_pvp_virtio_user_4k_pages.py b/tests/TestSuite_pvp_virtio_user_4k_pages.py index 3e3e09b4..aa41cd50 100644 --- a/tests/TestSuite_pvp_virtio_user_4k_pages.py +++ b/tests/TestSuite_pvp_virtio_user_4k_pages.py @@ -2,16 +2,11 @@ # Copyright(c) 2019 Intel Corporation # -""" -DPDK Test suite. -vhost/virtio-user pvp with 4K pages. -""" +from copy import deepcopy -import time - -import framework.utils as utils from framework.packet import Packet from framework.pktgen import PacketGeneratorHelper +from framework.settings import UPDATE_EXPECTED, load_global_setting from framework.test_case import TestCase @@ -41,6 +36,8 @@ class TestPvpVirtioUser4kPages(TestCase): self.core_list_virtio_user = self.core_list[0:2] self.core_list_vhost_user = self.core_list[2:4] self.pci_info = self.dut.ports_info[0]["pci"] + self.nb_ports = 1 + self.gap = self.get_suite_cfg()["accepted_tolerance"] self.dst_mac = self.dut.get_mac_address(self.dut_ports[0]) self.frame_sizes = [64, 128, 256, 512, 1024, 1518] self.logger.info( @@ -68,35 +65,19 @@ class TestPvpVirtioUser4kPages(TestCase): self.dut.send_expect("killall -s INT %s" % self.testpmd_name, "# ") self.vhost_user = self.dut.new_session(suite="vhost-user") self.virtio_user = self.dut.new_session(suite="virtio-user") + self.throughput = dict() + self.test_result = dict() # Prepare the result table self.table_header = ["Frame"] - self.table_header.append("Mode") self.table_header.append("Mpps") - self.table_header.append("Queue Num") self.table_header.append("% linerate") - self.result_table_create(self.table_header) - @property - def check_value(self): - check_dict = dict.fromkeys(self.frame_sizes) - linerate = { - 64: 0.08, - 128: 0.10, - 256: 0.18, - 512: 0.20, - 1024: 0.40, - 1280: 0.45, - 1518: 0.50, - } - for size in self.frame_sizes: - speed = self.wirespeed(self.nic, size, self.number_of_ports) - check_dict[size] = round(speed * linerate[size], 2) - return check_dict - - def send_and_verify(self): - """ - Send packet with packet generator and verify + def perf_test(self): + """ + Send packet with packet generator """ + self.result_table_create(self.table_header) + self.throughput = {} for frame_size in self.frame_sizes: tgen_input = [] rx_port = self.tester.get_local_port(self.dut_ports[0]) @@ -110,21 +91,71 @@ class TestPvpVirtioUser4kPages(TestCase): streams = self.pktgen_helper.prepare_stream_from_tginput( tgen_input, 100, None, self.tester.pktgen ) - _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams) - Mpps = pps / 1000000.0 - self.verify( - Mpps > self.check_value[frame_size], - "%s of frame size %d speed verify failed, expect %s, result %s" - % (self.running_case, frame_size, self.check_value[frame_size], Mpps), + # set traffic option + traffic_opt = { + "delay": 5, + "duration": self.get_suite_cfg()["test_duration"], + } + _, pps = self.tester.pktgen.measure_throughput( + stream_ids=streams, options=traffic_opt ) - throughput = Mpps * 100 / float(self.wirespeed(self.nic, 64, 1)) - + Mpps = pps / 1000000.0 + line_rate = Mpps * 100 / float(self.wirespeed(self.nic, 64, 1)) + self.throughput[frame_size] = Mpps results_row = [frame_size] - results_row.append("4K pages") results_row.append(Mpps) - results_row.append("1") - results_row.append(throughput) + results_row.append(line_rate) self.result_table_add(results_row) + self.result_table_print() + + def handle_expected(self): + """ + Update expected numbers to configurate file: $DTS_CFG_FOLDER/$suite_name.cfg + """ + if load_global_setting(UPDATE_EXPECTED) == "yes": + for frame_size in self.frame_sizes: + self.expected_throughput[frame_size] = round( + self.throughput[frame_size], 3 + ) + + def handle_results(self): + """ + results handled process: + 1, save to self.test_results + 2, create test results table + """ + # save test results to self.test_result + header = self.table_header + header.append("Expected Throughput(Mpps)") + header.append("Status") + self.result_table_create(self.table_header) + for frame_size in self.frame_sizes: + wirespeed = self.wirespeed(self.nic, frame_size, self.nb_ports) + ret_data = {} + ret_data[header[0]] = str(frame_size) + _real = float(self.throughput[frame_size]) + _exp = float(self.expected_throughput[frame_size]) + ret_data[header[1]] = "{:.3f}".format(_real) + ret_data[header[2]] = "{:.3f}%".format(_real * 100 / wirespeed) + ret_data[header[3]] = "{:.3f}".format(_exp) + gap = _exp * -self.gap * 0.01 + if _real > _exp + gap: + ret_data[header[4]] = "PASS" + else: + ret_data[header[4]] = "FAIL" + self.test_result[frame_size] = deepcopy(ret_data) + + for frame_size in self.test_result.keys(): + table_row = list() + for i in range(len(header)): + table_row.append(self.test_result[frame_size][header[i]]) + self.result_table_add(table_row) + # present test results to screen + self.result_table_print() + self.verify( + "FAIL" not in self.test_result, + "Excessive gap between test results and expectations", + ) def start_testpmd_as_vhost(self): """ @@ -187,22 +218,32 @@ class TestPvpVirtioUser4kPages(TestCase): """ Basic test for virtio-user 4K pages """ + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()["expected_throughput"][ + self.test_target + ] self.start_testpmd_as_vhost() self.prepare_tmpfs_for_4k() self.start_testpmd_as_virtio() - self.send_and_verify() - self.result_table_print() + self.perf_test() + self.handle_expected() + self.handle_results() self.close_all_apps() def test_perf_pvp_virtio_user_packed_ring_with_4K_pages(self): """ Basic test for virtio-user 4K pages """ + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()["expected_throughput"][ + self.test_target + ] self.start_testpmd_as_vhost() self.prepare_tmpfs_for_4k() self.start_testpmd_as_virtio(packed=True) - self.send_and_verify() - self.result_table_print() + self.perf_test() + self.handle_expected() + self.handle_results() self.close_all_apps() def tear_down(self): -- 2.25.1