1. Sanity Script is enhanced with additional test cases being added to autotest_data.py 2. Fixed in autotest_test_funcs.py to handle test cases which returns "Skipped" as result. The issue was skipped test cases got timed out, causing delay in sanity script execution. 3. Enhanced support for FreeBSD to add the respective memory limit, as FreeBSD doesn't support socket-mem, file-prefix options. Signed-off-by: Jananee Parthasarathy <jananeex.m.parthasarathy@intel.com> --- test/test/autotest_data.py | 471 ++++++++++++++++++++++++++++++++++++++- test/test/autotest_runner.py | 9 +- test/test/autotest_test_funcs.py | 6 +- 3 files changed, 474 insertions(+), 12 deletions(-) mode change 100644 => 100755 test/test/autotest_data.py mode change 100644 => 100755 test/test/autotest_test_funcs.py diff --git a/test/test/autotest_data.py b/test/test/autotest_data.py old mode 100644 new mode 100755 index aacfe0a..2c6828d --- a/test/test/autotest_data.py +++ b/test/test/autotest_data.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2010-2018 Intel Corporation # Test data for autotests @@ -70,6 +70,31 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Eventdev selftest octeontx", + "Command": "eventdev_selftest_octeontx", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Event ring autotest", + "Command": "event_ring_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Table autotest", + "Command": "table_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Flow classify autotest", + "Command": "flow_classify_autotest", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -125,6 +150,50 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Dump struct sizes", + "Command": "dump_struct_sizes", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump mempool", + "Command": "dump_mempool", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump malloc stats", + "Command": "dump_malloc_stats", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump devargs", + "Command": "dump_devargs", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump log types", + "Command": "dump_log_types", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump_ring", + "Command": "dump_ring", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Quit", + "Command": "quit", + "Func": default_autotest, + "Report": None, + }, + + ], }, { @@ -168,6 +237,26 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Set rxtx mode", + "Command": "set_rxtx_mode", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Set rxtx anchor", + "Command": "set_rxtx_anchor", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Set rxtx sc", + "Command": "set_rxtx_sc", + "Func": default_autotest, + "Report": None, + }, + + ] }, { @@ -176,12 +265,6 @@ def per_sockets(num): "Tests": [ { - "Name": "PCI autotest", - "Command": "pci_autotest", - "Func": default_autotest, - "Report": None, - }, - { "Name": "Malloc autotest", "Command": "malloc_autotest", "Func": default_autotest, @@ -211,6 +294,37 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Event eth rx adapter autotest", + "Command": "event_eth_rx_adapter_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Rawdev autotest", + "Command": "rawdev_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Kvargs autotest", + "Command": "kvargs_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump physmem", + "Command": "dump_physmem", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump memzone", + "Command": "dump_memzone", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -248,6 +362,31 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Devargs autotest", + "Command": "devargs_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Link bonding autotest", + "Command": "link_bonding_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Link bonding mode4 autotest", + "Command": "link_bonding_mode4_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Link bonding rssconf autotest", + "Command": "link_bonding_rssconf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -285,6 +424,25 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Crc autotest", + "Command": "crc_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Distributor autotest", + "Command": "distributor_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Reorder autotest", + "Command": "reorder_autotest", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -310,8 +468,184 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Barrier autotest", + "Command": "barrier_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "User delay", + "Command": "user_delay_us", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Bitmap test", + "Command": "bitmap_test", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Hash scaling autotest", + "Command": "hash_scaling_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Hash multiwriter autotest", + "Command": "hash_multiwriter_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Service autotest", + "Command": "service_autotest", + "Func": default_autotest, + "Report": None, + }, + + + ] + }, + { + "Prefix": "group_8", + "Memory": "128", + "Tests": + [ + { + "Name": "Timer racecond autotest", + "Command": "timer_racecond_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Member autotest", + "Command": "member_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Efd_autotest", + "Command": "efd_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Thash autotest", + "Command": "thash_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Hash function autotest", + "Command": "hash_functions_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "group_9", + "Memory": "64", + "Tests": + [ + { + "Name": "Cryptodev sw mrvl autotest", + "Command": "cryptodev_sw_mrvl_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev dpaa2 sec autotest", + "Command": "cryptodev_dpaa2_sec_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev dpaa sec autotest", + "Command": "cryptodev_dpaa_sec_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev qat autotest", + "Command": "cryptodev_qat_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev aesni mb autotest", + "Command": "cryptodev_aesni_mb_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev openssl autotest", + "Command": "cryptodev_openssl_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev scheduler autotest", + "Command": "cryptodev_scheduler_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev aesni gcm autotest", + "Command": "cryptodev_aesni_gcm_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev null autotest", + "Command": "cryptodev_null_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev sw snow3g autotest", + "Command": "cryptodev_sw_snow3g_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev sw kasumi autotest", + "Command": "cryptodev_sw_kasumi_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev_sw_zuc_autotest", + "Command": "cryptodev_sw_zuc_autotest", + "Func": default_autotest, + "Report": None, + }, + + ] }, + { + "Prefix": "group_10", + "Memory": "64", + "Tests": + [ + { + "Name": "Reciprocal division", + "Command": "reciprocal_division", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Red all", + "Command": "red_all", + "Func": default_autotest, + "Report": None, + }, + ] + }, + ] # tests that should not be run when any other tests are running @@ -336,8 +670,8 @@ def per_sockets(num): "Tests": [ { - "Name": "Eventdev sw autotest", - "Command": "eventdev_sw_autotest", + "Name": "Eventdev selftest sw", + "Command": "eventdev_selftest_sw", "Func": default_autotest, "Report": None, }, @@ -447,6 +781,125 @@ def per_sockets(num): }, ] }, + { + "Prefix": "reciprocal_division_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Reciprocal division perf", + "Command": "reciprocal_division_perf", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "member_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Member perf autotest", + "Command": "member_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "efd_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Efd perf autotest", + "Command": "efd_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "lpm_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Lpm perf autotest", + "Command": "lpm_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "lpm6_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Lpm6 perf autotest", + "Command": "lpm6_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "red_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Red_perf", + "Command": "red_perf", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "distributor_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Distributor perf autotest", + "Command": "distributor_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "ring_pmd_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Ring pmd perf autotest", + "Command": "ring_pmd_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "pmd_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Pmd perf autotest", + "Command": "pmd_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + + # # Please always make sure that ring_perf is the last test! diff --git a/test/test/autotest_runner.py b/test/test/autotest_runner.py index a692f06..77f0ac7 100644 --- a/test/test/autotest_runner.py +++ b/test/test/autotest_runner.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2010-2018 Intel Corporation # The main logic behind running autotests in parallel @@ -201,6 +201,13 @@ def __init__(self, cmdline, target, blacklist, whitelist): # set up cmdline string def __get_cmdline(self, test): cmdline = self.cmdline + os_name = platform.system() + + # append the respective memory limitation on FREEBSD + # as socket-mem, file-prefix is not supported + if os_name == "FreeBSD" : + cmdline += " -m " + str(sum(map(int, test["Memory"].split(",")))) + return cmdline # append memory limitations for each test # otherwise tests won't run in parallel diff --git a/test/test/autotest_test_funcs.py b/test/test/autotest_test_funcs.py old mode 100644 new mode 100755 index 65fe335..219c208 --- a/test/test/autotest_test_funcs.py +++ b/test/test/autotest_test_funcs.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2010-2018 Intel Corporation # Test functions @@ -12,12 +12,14 @@ def default_autotest(child, test_name): child.sendline(test_name) result = child.expect(["Test OK", "Test Failed", - "Command not found", pexpect.TIMEOUT], timeout=900) + "Command not found", "Skipped", pexpect.TIMEOUT], timeout=900) if result == 1: return -1, "Fail" elif result == 2: return -1, "Fail [Not found]" elif result == 3: + return -1, "Fail [Test returns Skipped]" + elif result == 4: return -1, "Fail [Timeout]" return 0, "Success" -- 1.7.12.2
On 27-Apr-18 2:20 PM, Reshma Pattan wrote:
> 1. Sanity Script is enhanced with additional test cases
> being added to autotest_data.py
> 2. Fixed in autotest_test_funcs.py to handle test cases
> which returns "Skipped" as result.
> The issue was skipped test cases got timed out,
> causing delay in sanity script execution.
> 3. Enhanced support for FreeBSD to add the respective memory limit,
> as FreeBSD doesn't support socket-mem, file-prefix options.
>
> Signed-off-by: Jananee Parthasarathy <jananeex.m.parthasarathy@intel.com>
> ---
I don't think number 3 on that list is needed. FreeBSD doesn't support
file-prefixes, so it's not possible to run multiple unit tests in
parallel in the first place, so no need for grouping and no need to
limit allocated memory.
On a more general note, IMO this whole thing needs to be redone and
simplified. I wanted to fix all this as part of Memory Hotplug patches,
but didn't get to it yet.
(not to mention that this infrastructure itself is a lot like IVSHMEM,
in that it was built to solve a specific problem, consequences be damned...)
The reasoning behind splitting tests into groups was largely the fact
that we had to preallocate memory, while also running tests in parallel.
This is no longer needed for two reasons.
First, most (if not all) tests now clean up properly after themselves
(when this was written, it wasn't possible to un-reserve memzones, rings
etc. - now it is), so we can run them one by one without restarting the
test application.
Second, memory is now allocated on the fly, so there is no more reason
to split autotests into groups at all. Instead, we can just group all
tests into "performance" (to be run one by one) and "non performance"
(can be run concurrently via different prefixes), and run them with a
pool of workers.
For FreeBSD, since there's no dynamic memory allocation and no support
for file prefixes, there needs to be only one worker, running all tests
one by one. For Linux, you can add all unit tests into a queue, and use
a pool of workers dequeuing and doing tests in parallel under different
prefixes. No '-m' or '--socket-mem' switches needed.
--
Thanks,
Anatoly
On 27-Apr-18 2:20 PM, Reshma Pattan wrote: > 1. Sanity Script is enhanced with additional test cases > being added to autotest_data.py > 2. Fixed in autotest_test_funcs.py to handle test cases > which returns "Skipped" as result. > The issue was skipped test cases got timed out, > causing delay in sanity script execution. > 3. Enhanced support for FreeBSD to add the respective memory limit, > as FreeBSD doesn't support socket-mem, file-prefix options. > > Signed-off-by: Jananee Parthasarathy <jananeex.m.parthasarathy@intel.com> > --- <...> > @@ -1,5 +1,5 @@ > # SPDX-License-Identifier: BSD-3-Clause > -# Copyright(c) 2010-2014 Intel Corporation > +# Copyright(c) 2010-2018 Intel Corporation > > # The main logic behind running autotests in parallel > > @@ -201,6 +201,13 @@ def __init__(self, cmdline, target, blacklist, whitelist): > # set up cmdline string > def __get_cmdline(self, test): > cmdline = self.cmdline > + os_name = platform.system() > + > + # append the respective memory limitation on FREEBSD > + # as socket-mem, file-prefix is not supported > + if os_name == "FreeBSD" : > + cmdline += " -m " + str(sum(map(int, test["Memory"].split(",")))) > + return cmdline A better fix would've been to completely avoid calling __get_cmdline() for FreeBSD. Since file prefixes are not supported, tests cannot be run in parallel, so there is no need to limit allocated memory in the first place, so neither -m nor --socket-mem switches need to be added either. And without those, there's no need to modify the test cmdline at all. -- Thanks, Anatoly
Hi Anatoly, As per the comment " Since file prefixes are not supported, tests cannot be run in parallel" Does this means we need not run parallel test group test cases for FreeBSD. So the total test count for FreeBSD will be based on Non-Parallel test groups only. Please confirm / share your views on the same. Thanks M.P.Jananee -----Original Message----- From: Burakov, Anatoly Sent: Friday, April 27, 2018 7:26 PM To: Pattan, Reshma <reshma.pattan@intel.com>; Mcnamara, John <john.mcnamara@intel.com>; dev@dpdk.org Cc: Parthasarathy, JananeeX M <jananeex.m.parthasarathy@intel.com> Subject: Re: [dpdk-dev] [PATCH] app/test: enhance sanity script On 27-Apr-18 2:20 PM, Reshma Pattan wrote: > 1. Sanity Script is enhanced with additional test cases being added to > autotest_data.py 2. Fixed in autotest_test_funcs.py to handle test > cases which returns "Skipped" as result. > The issue was skipped test cases got timed out, causing delay in > sanity script execution. > 3. Enhanced support for FreeBSD to add the respective memory limit, as > FreeBSD doesn't support socket-mem, file-prefix options. > > Signed-off-by: Jananee Parthasarathy > <jananeex.m.parthasarathy@intel.com> > --- <...> > @@ -1,5 +1,5 @@ > # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 > Intel Corporation > +# Copyright(c) 2010-2018 Intel Corporation > > # The main logic behind running autotests in parallel > > @@ -201,6 +201,13 @@ def __init__(self, cmdline, target, blacklist, whitelist): > # set up cmdline string > def __get_cmdline(self, test): > cmdline = self.cmdline > + os_name = platform.system() > + > + # append the respective memory limitation on FREEBSD > + # as socket-mem, file-prefix is not supported > + if os_name == "FreeBSD" : > + cmdline += " -m " + str(sum(map(int, test["Memory"].split(",")))) > + return cmdline A better fix would've been to completely avoid calling __get_cmdline() for FreeBSD. Since file prefixes are not supported, tests cannot be run in parallel, so there is no need to limit allocated memory in the first place, so neither -m nor --socket-mem switches need to be added either. And without those, there's no need to modify the test cmdline at all. -- Thanks, Anatoly -------------------------------------------------------------- Intel Research and Development Ireland Limited Registered in Ireland Registered Office: Collinstown Industrial Park, Leixlip, County Kildare Registered Number: 308263 This e-mail and any attachments may contain confidential material for the sole use of the intended recipient(s). Any review or distribution by others is strictly prohibited. If you are not the intended recipient, please contact the sender and delete all copies.
On 03-May-18 11:03 AM, Parthasarathy, JananeeX M wrote: > Hi Anatoly, > > As per the comment " Since file prefixes are not supported, tests cannot be run in parallel" > Does this means we need not run parallel test group test cases for FreeBSD. So the total test count for FreeBSD will be based on Non-Parallel test groups only. No, the tests in the "parallel group" can be run in parallel, but on FreeBSD you cannot run them *in parallel* - that doesn't mean you don't have to run them :) Just ensure that the thread pool size is set to 1 on FreeBSD. > > Please confirm / share your views on the same. > > Thanks > M.P.Jananee > > -----Original Message----- > From: Burakov, Anatoly > Sent: Friday, April 27, 2018 7:26 PM > To: Pattan, Reshma <reshma.pattan@intel.com>; Mcnamara, John <john.mcnamara@intel.com>; dev@dpdk.org > Cc: Parthasarathy, JananeeX M <jananeex.m.parthasarathy@intel.com> > Subject: Re: [dpdk-dev] [PATCH] app/test: enhance sanity script > > On 27-Apr-18 2:20 PM, Reshma Pattan wrote: >> 1. Sanity Script is enhanced with additional test cases being added to >> autotest_data.py 2. Fixed in autotest_test_funcs.py to handle test >> cases which returns "Skipped" as result. >> The issue was skipped test cases got timed out, causing delay in >> sanity script execution. >> 3. Enhanced support for FreeBSD to add the respective memory limit, as >> FreeBSD doesn't support socket-mem, file-prefix options. >> >> Signed-off-by: Jananee Parthasarathy >> <jananeex.m.parthasarathy@intel.com> >> --- > > <...> > >> @@ -1,5 +1,5 @@ >> # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 >> Intel Corporation >> +# Copyright(c) 2010-2018 Intel Corporation >> >> # The main logic behind running autotests in parallel >> >> @@ -201,6 +201,13 @@ def __init__(self, cmdline, target, blacklist, whitelist): >> # set up cmdline string >> def __get_cmdline(self, test): >> cmdline = self.cmdline >> + os_name = platform.system() >> + >> + # append the respective memory limitation on FREEBSD >> + # as socket-mem, file-prefix is not supported >> + if os_name == "FreeBSD" : >> + cmdline += " -m " + str(sum(map(int, test["Memory"].split(",")))) >> + return cmdline > > A better fix would've been to completely avoid calling __get_cmdline() for FreeBSD. Since file prefixes are not supported, tests cannot be run in parallel, so there is no need to limit allocated memory in the first place, so neither -m nor --socket-mem switches need to be added either. > And without those, there's no need to modify the test cmdline at all. Also, regarding this - i think what i suggested earlier is slightly incorrect. A better fix would be to simply return cmdline without modifications on FreeBSD. Just because we don't modify cmdline now doesn't mean we wouldn't be doing so in the future, so not calling __get_cmdline is not the best solution. > > -- > Thanks, > Anatoly > -- Thanks, Anatoly
1. Sanity Script is enhanced with additional test cases being added to autotest_data.py 2. Fixed in autotest_test_funcs.py to handle test cases which returns "Skipped" as result. The issue was skipped test cases got timed out, causing delay in sanity script execution. 3. Enhanced support for FreeBSD, as FreeBSD doesn't support socket-mem, file-prefix options. Signed-off-by: Jananee Parthasarathy <jananeex.m.parthasarathy@intel.com> Reviewed-by: Reshma Pattan <reshma.pattan@intel.com> Reviewed-by: Anatoly Burakov <anatoly.burakov@intel.com> --- v2: test command-line updated for freebsd --- test/test/autotest_data.py | 471 ++++++++++++++++++++++++++++++++++++++- test/test/autotest_runner.py | 8 +- test/test/autotest_test_funcs.py | 6 +- 3 files changed, 473 insertions(+), 12 deletions(-) mode change 100644 => 100755 test/test/autotest_data.py mode change 100644 => 100755 test/test/autotest_test_funcs.py diff --git a/test/test/autotest_data.py b/test/test/autotest_data.py old mode 100644 new mode 100755 index aacfe0a..2c6828d --- a/test/test/autotest_data.py +++ b/test/test/autotest_data.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2010-2018 Intel Corporation # Test data for autotests @@ -70,6 +70,31 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Eventdev selftest octeontx", + "Command": "eventdev_selftest_octeontx", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Event ring autotest", + "Command": "event_ring_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Table autotest", + "Command": "table_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Flow classify autotest", + "Command": "flow_classify_autotest", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -125,6 +150,50 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Dump struct sizes", + "Command": "dump_struct_sizes", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump mempool", + "Command": "dump_mempool", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump malloc stats", + "Command": "dump_malloc_stats", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump devargs", + "Command": "dump_devargs", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump log types", + "Command": "dump_log_types", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump_ring", + "Command": "dump_ring", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Quit", + "Command": "quit", + "Func": default_autotest, + "Report": None, + }, + + ], }, { @@ -168,6 +237,26 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Set rxtx mode", + "Command": "set_rxtx_mode", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Set rxtx anchor", + "Command": "set_rxtx_anchor", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Set rxtx sc", + "Command": "set_rxtx_sc", + "Func": default_autotest, + "Report": None, + }, + + ] }, { @@ -176,12 +265,6 @@ def per_sockets(num): "Tests": [ { - "Name": "PCI autotest", - "Command": "pci_autotest", - "Func": default_autotest, - "Report": None, - }, - { "Name": "Malloc autotest", "Command": "malloc_autotest", "Func": default_autotest, @@ -211,6 +294,37 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Event eth rx adapter autotest", + "Command": "event_eth_rx_adapter_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Rawdev autotest", + "Command": "rawdev_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Kvargs autotest", + "Command": "kvargs_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump physmem", + "Command": "dump_physmem", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump memzone", + "Command": "dump_memzone", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -248,6 +362,31 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Devargs autotest", + "Command": "devargs_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Link bonding autotest", + "Command": "link_bonding_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Link bonding mode4 autotest", + "Command": "link_bonding_mode4_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Link bonding rssconf autotest", + "Command": "link_bonding_rssconf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -285,6 +424,25 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Crc autotest", + "Command": "crc_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Distributor autotest", + "Command": "distributor_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Reorder autotest", + "Command": "reorder_autotest", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -310,8 +468,184 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Barrier autotest", + "Command": "barrier_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "User delay", + "Command": "user_delay_us", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Bitmap test", + "Command": "bitmap_test", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Hash scaling autotest", + "Command": "hash_scaling_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Hash multiwriter autotest", + "Command": "hash_multiwriter_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Service autotest", + "Command": "service_autotest", + "Func": default_autotest, + "Report": None, + }, + + + ] + }, + { + "Prefix": "group_8", + "Memory": "128", + "Tests": + [ + { + "Name": "Timer racecond autotest", + "Command": "timer_racecond_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Member autotest", + "Command": "member_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Efd_autotest", + "Command": "efd_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Thash autotest", + "Command": "thash_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Hash function autotest", + "Command": "hash_functions_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "group_9", + "Memory": "64", + "Tests": + [ + { + "Name": "Cryptodev sw mrvl autotest", + "Command": "cryptodev_sw_mrvl_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev dpaa2 sec autotest", + "Command": "cryptodev_dpaa2_sec_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev dpaa sec autotest", + "Command": "cryptodev_dpaa_sec_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev qat autotest", + "Command": "cryptodev_qat_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev aesni mb autotest", + "Command": "cryptodev_aesni_mb_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev openssl autotest", + "Command": "cryptodev_openssl_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev scheduler autotest", + "Command": "cryptodev_scheduler_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev aesni gcm autotest", + "Command": "cryptodev_aesni_gcm_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev null autotest", + "Command": "cryptodev_null_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev sw snow3g autotest", + "Command": "cryptodev_sw_snow3g_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev sw kasumi autotest", + "Command": "cryptodev_sw_kasumi_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev_sw_zuc_autotest", + "Command": "cryptodev_sw_zuc_autotest", + "Func": default_autotest, + "Report": None, + }, + + ] }, + { + "Prefix": "group_10", + "Memory": "64", + "Tests": + [ + { + "Name": "Reciprocal division", + "Command": "reciprocal_division", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Red all", + "Command": "red_all", + "Func": default_autotest, + "Report": None, + }, + ] + }, + ] # tests that should not be run when any other tests are running @@ -336,8 +670,8 @@ def per_sockets(num): "Tests": [ { - "Name": "Eventdev sw autotest", - "Command": "eventdev_sw_autotest", + "Name": "Eventdev selftest sw", + "Command": "eventdev_selftest_sw", "Func": default_autotest, "Report": None, }, @@ -447,6 +781,125 @@ def per_sockets(num): }, ] }, + { + "Prefix": "reciprocal_division_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Reciprocal division perf", + "Command": "reciprocal_division_perf", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "member_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Member perf autotest", + "Command": "member_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "efd_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Efd perf autotest", + "Command": "efd_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "lpm_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Lpm perf autotest", + "Command": "lpm_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "lpm6_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Lpm6 perf autotest", + "Command": "lpm6_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "red_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Red_perf", + "Command": "red_perf", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "distributor_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Distributor perf autotest", + "Command": "distributor_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "ring_pmd_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Ring pmd perf autotest", + "Command": "ring_pmd_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "pmd_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Pmd perf autotest", + "Command": "pmd_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + + # # Please always make sure that ring_perf is the last test! diff --git a/test/test/autotest_runner.py b/test/test/autotest_runner.py index a692f06..0997e78 100644 --- a/test/test/autotest_runner.py +++ b/test/test/autotest_runner.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2010-2018 Intel Corporation # The main logic behind running autotests in parallel @@ -201,6 +201,12 @@ def __init__(self, cmdline, target, blacklist, whitelist): # set up cmdline string def __get_cmdline(self, test): cmdline = self.cmdline + os_name = sys.platform + + # test command line need not be modified for FreeBSD + # as socket-mem, file-prefix is not supported + if os_name.startswith('freebsd') : + return cmdline # append memory limitations for each test # otherwise tests won't run in parallel diff --git a/test/test/autotest_test_funcs.py b/test/test/autotest_test_funcs.py old mode 100644 new mode 100755 index 65fe335..219c208 --- a/test/test/autotest_test_funcs.py +++ b/test/test/autotest_test_funcs.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2010-2018 Intel Corporation # Test functions @@ -12,12 +12,14 @@ def default_autotest(child, test_name): child.sendline(test_name) result = child.expect(["Test OK", "Test Failed", - "Command not found", pexpect.TIMEOUT], timeout=900) + "Command not found", "Skipped", pexpect.TIMEOUT], timeout=900) if result == 1: return -1, "Fail" elif result == 2: return -1, "Fail [Not found]" elif result == 3: + return -1, "Fail [Test returns Skipped]" + elif result == 4: return -1, "Fail [Timeout]" return 0, "Success" -- 1.7.12.2
Hi, 10/05/2018 12:36, Jananee Parthasarathy: > 1. Sanity Script is enhanced with additional test cases > being added to autotest_data.py How long they are? How did you choose to group them? > 2. Fixed in autotest_test_funcs.py to handle test cases > which returns "Skipped" as result. > The issue was skipped test cases got timed out, > causing delay in sanity script execution. > 3. Enhanced support for FreeBSD, > as FreeBSD doesn't support socket-mem, file-prefix options. Please do 3 patches.
Hi Thomas,
> > 2. Fixed in autotest_test_funcs.py to handle test cases which returns
> > "Skipped" as result.
> > The issue was skipped test cases got timed out, causing delay in
> > sanity script execution.
> > 3. Enhanced support for FreeBSD,
> > as FreeBSD doesn't support socket-mem, file-prefix options.
>
> Please do 3 patches.
Why do we need 3 patches here? Any reason?
Thanks,
Reshma
14/05/2018 17:26, Pattan, Reshma:
> Hi Thomas,
>
> > > 2. Fixed in autotest_test_funcs.py to handle test cases which returns
> > > "Skipped" as result.
> > > The issue was skipped test cases got timed out, causing delay in
> > > sanity script execution.
> > > 3. Enhanced support for FreeBSD,
> > > as FreeBSD doesn't support socket-mem, file-prefix options.
> >
> > Please do 3 patches.
>
> Why do we need 3 patches here? Any reason?
Because it's easier to review and better for the history to have
patches with only one goal.
It is a basic rule: one fix = one patch.
It is also easier to backport.
1/3: new test cases are added to sanity script 2/3: skipped test case result is updated appropriately 3/3: test cmdline is updated to support freebsd Signed-off-by: Jananee Parthasarathy <jananeex.m.parthasarathy@intel.com> Reviewed-by: Reshma Pattan <reshma.pattan@intel.com> Reviewed-by: Anatoly Burakov <anatoly.burakov@intel.com> --- v3: created separate patches for all sanity script changes --- Jananee Parthasarathy (3): app/test: enhance sanity script app/test: update result for skipped test cases app/test: enhance freebsd support in sanity script test/test/autotest_data.py | 471 ++++++++++++++++++++++++++++++++++++++- test/test/autotest_runner.py | 8 +- test/test/autotest_test_funcs.py | 6 +- 3 files changed, 473 insertions(+), 12 deletions(-) mode change 100644 => 100755 test/test/autotest_data.py mode change 100644 => 100755 test/test/autotest_test_funcs.py -- 1.7.12.2
Sanity Script is enhanced with additional test cases being added to autotest_data.py Signed-off-by: Jananee Parthasarathy <jananeex.m.parthasarathy@intel.com> --- test/test/autotest_data.py | 471 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 462 insertions(+), 9 deletions(-) mode change 100644 => 100755 test/test/autotest_data.py diff --git a/test/test/autotest_data.py b/test/test/autotest_data.py old mode 100644 new mode 100755 index aacfe0a..2c6828d --- a/test/test/autotest_data.py +++ b/test/test/autotest_data.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2010-2018 Intel Corporation # Test data for autotests @@ -70,6 +70,31 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Eventdev selftest octeontx", + "Command": "eventdev_selftest_octeontx", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Event ring autotest", + "Command": "event_ring_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Table autotest", + "Command": "table_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Flow classify autotest", + "Command": "flow_classify_autotest", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -125,6 +150,50 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Dump struct sizes", + "Command": "dump_struct_sizes", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump mempool", + "Command": "dump_mempool", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump malloc stats", + "Command": "dump_malloc_stats", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump devargs", + "Command": "dump_devargs", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump log types", + "Command": "dump_log_types", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump_ring", + "Command": "dump_ring", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Quit", + "Command": "quit", + "Func": default_autotest, + "Report": None, + }, + + ], }, { @@ -168,6 +237,26 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Set rxtx mode", + "Command": "set_rxtx_mode", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Set rxtx anchor", + "Command": "set_rxtx_anchor", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Set rxtx sc", + "Command": "set_rxtx_sc", + "Func": default_autotest, + "Report": None, + }, + + ] }, { @@ -176,12 +265,6 @@ def per_sockets(num): "Tests": [ { - "Name": "PCI autotest", - "Command": "pci_autotest", - "Func": default_autotest, - "Report": None, - }, - { "Name": "Malloc autotest", "Command": "malloc_autotest", "Func": default_autotest, @@ -211,6 +294,37 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Event eth rx adapter autotest", + "Command": "event_eth_rx_adapter_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Rawdev autotest", + "Command": "rawdev_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Kvargs autotest", + "Command": "kvargs_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump physmem", + "Command": "dump_physmem", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump memzone", + "Command": "dump_memzone", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -248,6 +362,31 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Devargs autotest", + "Command": "devargs_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Link bonding autotest", + "Command": "link_bonding_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Link bonding mode4 autotest", + "Command": "link_bonding_mode4_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Link bonding rssconf autotest", + "Command": "link_bonding_rssconf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -285,6 +424,25 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Crc autotest", + "Command": "crc_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Distributor autotest", + "Command": "distributor_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Reorder autotest", + "Command": "reorder_autotest", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -310,8 +468,184 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Barrier autotest", + "Command": "barrier_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "User delay", + "Command": "user_delay_us", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Bitmap test", + "Command": "bitmap_test", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Hash scaling autotest", + "Command": "hash_scaling_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Hash multiwriter autotest", + "Command": "hash_multiwriter_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Service autotest", + "Command": "service_autotest", + "Func": default_autotest, + "Report": None, + }, + + + ] + }, + { + "Prefix": "group_8", + "Memory": "128", + "Tests": + [ + { + "Name": "Timer racecond autotest", + "Command": "timer_racecond_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Member autotest", + "Command": "member_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Efd_autotest", + "Command": "efd_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Thash autotest", + "Command": "thash_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Hash function autotest", + "Command": "hash_functions_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "group_9", + "Memory": "64", + "Tests": + [ + { + "Name": "Cryptodev sw mrvl autotest", + "Command": "cryptodev_sw_mrvl_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev dpaa2 sec autotest", + "Command": "cryptodev_dpaa2_sec_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev dpaa sec autotest", + "Command": "cryptodev_dpaa_sec_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev qat autotest", + "Command": "cryptodev_qat_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev aesni mb autotest", + "Command": "cryptodev_aesni_mb_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev openssl autotest", + "Command": "cryptodev_openssl_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev scheduler autotest", + "Command": "cryptodev_scheduler_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev aesni gcm autotest", + "Command": "cryptodev_aesni_gcm_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev null autotest", + "Command": "cryptodev_null_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev sw snow3g autotest", + "Command": "cryptodev_sw_snow3g_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev sw kasumi autotest", + "Command": "cryptodev_sw_kasumi_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev_sw_zuc_autotest", + "Command": "cryptodev_sw_zuc_autotest", + "Func": default_autotest, + "Report": None, + }, + + ] }, + { + "Prefix": "group_10", + "Memory": "64", + "Tests": + [ + { + "Name": "Reciprocal division", + "Command": "reciprocal_division", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Red all", + "Command": "red_all", + "Func": default_autotest, + "Report": None, + }, + ] + }, + ] # tests that should not be run when any other tests are running @@ -336,8 +670,8 @@ def per_sockets(num): "Tests": [ { - "Name": "Eventdev sw autotest", - "Command": "eventdev_sw_autotest", + "Name": "Eventdev selftest sw", + "Command": "eventdev_selftest_sw", "Func": default_autotest, "Report": None, }, @@ -447,6 +781,125 @@ def per_sockets(num): }, ] }, + { + "Prefix": "reciprocal_division_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Reciprocal division perf", + "Command": "reciprocal_division_perf", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "member_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Member perf autotest", + "Command": "member_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "efd_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Efd perf autotest", + "Command": "efd_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "lpm_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Lpm perf autotest", + "Command": "lpm_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "lpm6_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Lpm6 perf autotest", + "Command": "lpm6_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "red_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Red_perf", + "Command": "red_perf", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "distributor_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Distributor perf autotest", + "Command": "distributor_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "ring_pmd_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Ring pmd perf autotest", + "Command": "ring_pmd_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "pmd_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Pmd perf autotest", + "Command": "pmd_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + + # # Please always make sure that ring_perf is the last test! -- 1.7.12.2
Fixed in autotest_test_funcs.py to handle test cases which returns "Skipped" as result. The issue was skipped test cases got timed out, causing delay in sanity script execution. Signed-off-by: Jananee Parthasarathy <jananeex.m.parthasarathy@intel.com> --- test/test/autotest_test_funcs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) mode change 100644 => 100755 test/test/autotest_test_funcs.py diff --git a/test/test/autotest_test_funcs.py b/test/test/autotest_test_funcs.py old mode 100644 new mode 100755 index 65fe335..219c208 --- a/test/test/autotest_test_funcs.py +++ b/test/test/autotest_test_funcs.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2010-2018 Intel Corporation # Test functions @@ -12,12 +12,14 @@ def default_autotest(child, test_name): child.sendline(test_name) result = child.expect(["Test OK", "Test Failed", - "Command not found", pexpect.TIMEOUT], timeout=900) + "Command not found", "Skipped", pexpect.TIMEOUT], timeout=900) if result == 1: return -1, "Fail" elif result == 2: return -1, "Fail [Not found]" elif result == 3: + return -1, "Fail [Test returns Skipped]" + elif result == 4: return -1, "Fail [Timeout]" return 0, "Success" -- 1.7.12.2
Update test command line with no additional arguments, as FreeBSD doesn't support socket-mem, file-prefix options Signed-off-by: Jananee Parthasarathy <jananeex.m.parthasarathy@intel.com> --- test/test/autotest_runner.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/test/autotest_runner.py b/test/test/autotest_runner.py index a692f06..0997e78 100644 --- a/test/test/autotest_runner.py +++ b/test/test/autotest_runner.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2010-2018 Intel Corporation # The main logic behind running autotests in parallel @@ -201,6 +201,12 @@ def __init__(self, cmdline, target, blacklist, whitelist): # set up cmdline string def __get_cmdline(self, test): cmdline = self.cmdline + os_name = sys.platform + + # test command line need not be modified for FreeBSD + # as socket-mem, file-prefix is not supported + if os_name.startswith('freebsd') : + return cmdline # append memory limitations for each test # otherwise tests won't run in parallel -- 1.7.12.2
On 15-May-18 4:58 PM, Jananee Parthasarathy wrote:
> Sanity Script is enhanced with additional test cases
> being added to autotest_data.py
>
> Signed-off-by: Jananee Parthasarathy <jananeex.m.parthasarathy@intel.com>
> ---
Patchset-wide reviews/acks should be part of the commit message.
Just out of curiousity - where did the name "sanity script" come from? I
wonder why are you referring to autotest config that way...
--
Thanks,
Anatoly
Hi Thomas, -----Original Message----- From: Thomas Monjalon [mailto:thomas@monjalon.net] Sent: Monday, May 14, 2018 3:42 AM To: Parthasarathy, JananeeX M <jananeex.m.parthasarathy@intel.com> Cc: dev@dpdk.org; Burakov, Anatoly <anatoly.burakov@intel.com>; Pattan, Reshma <reshma.pattan@intel.com> Subject: Re: [dpdk-dev] [PATCH v2] app/test: enhance sanity script Hi, 10/05/2018 12:36, Jananee Parthasarathy: > 1. Sanity Script is enhanced with additional test cases being added to > autotest_data.py How long they are? How did you choose to group them? It took approximately ~30 min, but longer duration is due to failed test cases (15m) which times out due to crash. Grouping is done based on memory and time duration. > 2. Fixed in autotest_test_funcs.py to handle test cases which returns > "Skipped" as result. > The issue was skipped test cases got timed out, causing delay in > sanity script execution. > 3. Enhanced support for FreeBSD, > as FreeBSD doesn't support socket-mem, file-prefix options. Please do 3 patches. OK created as suggested. Regards M.P.Jananee -------------------------------------------------------------- Intel Research and Development Ireland Limited Registered in Ireland Registered Office: Collinstown Industrial Park, Leixlip, County Kildare Registered Number: 308263 This e-mail and any attachments may contain confidential material for the sole use of the intended recipient(s). Any review or distribution by others is strictly prohibited. If you are not the intended recipient, please contact the sender and delete all copies.
15/05/2018 18:14, Parthasarathy, JananeeX M:
> From: Thomas Monjalon [mailto:thomas@monjalon.net]
> > 10/05/2018 12:36, Jananee Parthasarathy:
> > > 1. Sanity Script is enhanced with additional test cases being added to
> > > autotest_data.py
> >
> > How long they are?
> > How did you choose to group them?
>
> It took approximately ~30 min, but longer duration is due to failed test cases (15m) which times out due to crash.
What was the duration of make test before your patch?
I think we need to keep a fast subset of tests.
Please check in mk/rte.sdktest.mk:
PERFLIST=ring_perf,mempool_perf,memcpy_perf,hash_perf,timer_perf
test-fast: BLACKLIST=-$(PERFLIST)
Hi Thomas,
-----Original Message-----
From: Thomas Monjalon [mailto:thomas@monjalon.net]
Sent: Tuesday, May 15, 2018 9:47 PM
To: Parthasarathy, JananeeX M <jananeex.m.parthasarathy@intel.com>
Cc: dev@dpdk.org; Burakov, Anatoly <anatoly.burakov@intel.com>; Pattan, Reshma <reshma.pattan@intel.com>
Subject: Re: [dpdk-dev] [PATCH v2] app/test: enhance sanity script
15/05/2018 18:14, Parthasarathy, JananeeX M:
> From: Thomas Monjalon [mailto:thomas@monjalon.net]
> > 10/05/2018 12:36, Jananee Parthasarathy:
> > > 1. Sanity Script is enhanced with additional test cases being
> > > added to autotest_data.py
> >
> > How long they are?
> > How did you choose to group them?
>
> It took approximately ~30 min, but longer duration is due to failed test cases (15m) which times out due to crash.
What was the duration of make test before your patch?
Using "make test-fast", total run time is 32s.
Earlier, we had run the tests directly using the python script in test/test folder as
Eg: ./autotest.py ./build/app/test may7_log
This has run all the test cases which took long time.
I think we need to keep a fast subset of tests.
Please check in mk/rte.sdktest.mk:
PERFLIST=ring_perf,mempool_perf,memcpy_perf,hash_perf,timer_perf
test-fast: BLACKLIST=-$(PERFLIST)
We have added the below test cases to blacklist in mk/rte.sdktest.mk
reciprocal_division,reciprocal_division_perf,lpm_perf,red_all
Now time taken is around 4mins.
Please let us know is there any minimum time limit for test execution.
Is this time duration fine?
Thanks
M.P.Jananee
--------------------------------------------------------------
Intel Research and Development Ireland Limited
Registered in Ireland
Registered Office: Collinstown Industrial Park, Leixlip, County Kildare
Registered Number: 308263
This e-mail and any attachments may contain confidential material for the sole
use of the intended recipient(s). Any review or distribution by others is
strictly prohibited. If you are not the intended recipient, please contact the
sender and delete all copies.
18/05/2018 15:32, Parthasarathy, JananeeX M:
> From: Thomas Monjalon [mailto:thomas@monjalon.net]
> > 15/05/2018 18:14, Parthasarathy, JananeeX M:
> > > From: Thomas Monjalon [mailto:thomas@monjalon.net]
> >
> > I think we need to keep a fast subset of tests.
> >
> > Please check in mk/rte.sdktest.mk:
> > PERFLIST=ring_perf,mempool_perf,memcpy_perf,hash_perf,timer_perf
> > test-fast: BLACKLIST=-$(PERFLIST)
>
> We have added the below test cases to blacklist in mk/rte.sdktest.mk
> reciprocal_division,reciprocal_division_perf,lpm_perf,red_all
>
> Now time taken is around 4mins.
> Please let us know is there any minimum time limit for test execution.
> Is this time duration fine?
4 minutes, it's a bit long for developer oriented tests.
We need to distinguish which tests are enough for quick tests,
and which ones can be in another category for full validation purpose.
Please can you make a summary of which tests you are adding,
and propose a priority classification.
Thanks
Hello Thomas, -----Original Message----- From: Thomas Monjalon [mailto:thomas@monjalon.net] Sent: Friday, May 18, 2018 7:18 PM To: Parthasarathy, JananeeX M <jananeex.m.parthasarathy@intel.com> Cc: dev@dpdk.org; Burakov, Anatoly <anatoly.burakov@intel.com>; Pattan, Reshma <reshma.pattan@intel.com> Subject: Re: [dpdk-dev] [PATCH v2] app/test: enhance sanity script 18/05/2018 15:32, Parthasarathy, JananeeX M: > From: Thomas Monjalon [mailto:thomas@monjalon.net] > > 15/05/2018 18:14, Parthasarathy, JananeeX M: > > > From: Thomas Monjalon [mailto:thomas@monjalon.net] > > > > I think we need to keep a fast subset of tests. > > > > Please check in mk/rte.sdktest.mk: > > PERFLIST=ring_perf,mempool_perf,memcpy_perf,hash_perf,timer_perf > > test-fast: BLACKLIST=-$(PERFLIST) > > We have added the below test cases to blacklist in mk/rte.sdktest.mk > reciprocal_division,reciprocal_division_perf,lpm_perf,red_all > > Now time taken is around 4mins. > Please let us know is there any minimum time limit for test execution. > Is this time duration fine? 4 minutes, it's a bit long for developer oriented tests. We need to distinguish which tests are enough for quick tests, and which ones can be in another category for full validation purpose. Please can you make a summary of which tests you are adding, and propose a priority classification. Thanks Please find below the test cases newly added to sanity script (autotest_data.py) with priority classification. P1 - Main test cases P2 - Cryptodev test cases which requires dependent libraries (Can be added if the environment is set accordingly) P3 - Perf test cases and others which takes more time (> 10s) (these will be in the blacklist test cases) P4 - Logging/Dump test cases (to be checked whether these are really necessary) The blacklist is updated in makefile "rte.sdktest.mk" with below list of test cases to reduce the total test execution time. Blacklist test cases are ring_perf,mempool_perf,memcpy_perf,hash_perf,timer_perf,reciprocal_division,reciprocal_division_perf,lpm_perf, red_all,barrier,hash_multiwriter,timer_racecond,efd,hash_functions,eventdev_selftest_sw,member_perf,efd_perf,lpm6_perf,red_perf,distributor_perf Currently after the above blacklist update, the total execution time of "make test-fast" is 54s. Eventdev selftest octeontx P1 Event ring autotest P1 Table autotest P1 Flow classify autotest P1 Dump struct sizes P4 Dump mempool P4 Dump malloc stats P4 Dump devargs P4 Dump log types P4 Dump_ring P4 Quit P4 Set rxtx anchor P4 Set rxtx sc P4 Event eth rx adapter autotest P1 Rawdev autotest P1 Kvargs autotest P1 Dump physmem P1 Dump memzone P4 Devargs autotest P4 Link bonding autotest P1 Link bonding mode4 autotest P1 Link bonding rssconf autotest P1 Crc autotest P1 Distributor autotest P1 Reorder autotest P1 Barrier autotest P3 User delay P1 Bitmap test P1 Hash scaling autotest P1 Hash multiwriter autotest P3 Service autotest P1 Timer racecond autotest P3 Member autotest P3 Efd_autotest P3 Thash autotest P1 Hash function autotest P3 Cryptodev sw mrvl autotest P2 Cryptodev dpaa2 sec autotest P2 Cryptodev dpaa sec autotest P2 Cryptodev qat autotest P2 Cryptodev aesni mb autotest P2 Cryptodev openssl autotest P2 Cryptodev scheduler autotest P2 Cryptodev aesni gcm autotest P2 Cryptodev null autotest P2 Cryptodev sw snow3g autotest P2 Cryptodev sw kasumi autotest P2 Cryptodev_sw_zuc_autotest P2 Reciprocal division P3 Red all P3 Mempool performance autotest P3 Memcpy performance autotest P3 Hash performance autotest P3 Timer performance autotest P3 Reciprocal division perf P3 Member perf autotest P3 Efd perf autotest P3 Lpm perf autotest P3 Lpm6 perf autotest P3 Red_perf P3 Distributor perf autotest P3 Ring pmd perf autotest P1 Pmd perf autotest P1 Ring performance autotest P1 Thanks M.P.Jananee -------------------------------------------------------------- Intel Research and Development Ireland Limited Registered in Ireland Registered Office: Collinstown Industrial Park, Leixlip, County Kildare Registered Number: 308263 This e-mail and any attachments may contain confidential material for the sole use of the intended recipient(s). Any review or distribution by others is strictly prohibited. If you are not the intended recipient, please contact the sender and delete all copies.
20/05/2018 19:41, Parthasarathy, JananeeX M: > From: Thomas Monjalon [mailto:thomas@monjalon.net] > > 18/05/2018 15:32, Parthasarathy, JananeeX M: > > > From: Thomas Monjalon [mailto:thomas@monjalon.net] > > > > 15/05/2018 18:14, Parthasarathy, JananeeX M: > > > > > From: Thomas Monjalon [mailto:thomas@monjalon.net] > > > > > > > > I think we need to keep a fast subset of tests. > > > > > > > > Please check in mk/rte.sdktest.mk: > > > > PERFLIST=ring_perf,mempool_perf,memcpy_perf,hash_perf,ti > > > > mer_perf > > > > > > > > test-fast: BLACKLIST=-$(PERFLIST) > > > > > > We have added the below test cases to blacklist in mk/rte.sdktest.mk > > > reciprocal_division,reciprocal_division_perf,lpm_perf,red_all > > > > > > Now time taken is around 4mins. > > > Please let us know is there any minimum time limit for test execution. > > > Is this time duration fine? > > > > 4 minutes, it's a bit long for developer oriented tests. > > > > We need to distinguish which tests are enough for quick tests, > > and which ones can be in another category for full validation purpose. > > > > Please can you make a summary of which tests you are adding, > > and propose a priority classification. > > > > Thanks > > Please find below the test cases newly added to sanity script (autotest_data.py) with priority classification. > > P1 - Main test cases We can move some tests from P1. Examples: - bonding can be P2 with other drivers. - PMD perf and ring perf can P3 with other perf tests. > P2 - Cryptodev test cases which requires dependent libraries (Can be added if the environment is set accordingly) > P3 - Perf test cases and others which takes more time (> 10s) (these will be in the blacklist test cases) > P4 - Logging/Dump test cases (to be checked whether these are really necessary) I think this classification is good. We can adjust the categories in the makefile to reflect it. The goal is to have a really quick and core subset (equal P1 category), and other categories available with a different make command (drivers for P2, perf for P3 and debug for P4). > The blacklist is updated in makefile "rte.sdktest.mk" with below list > of test cases to reduce the total test execution time. > > Blacklist test cases are ring_perf,mempool_perf,memcpy_perf,hash_perf,timer_perf, > reciprocal_division,reciprocal_division_perf,lpm_perf,red_all,barrier, > hash_multiwriter,timer_racecond,efd,hash_functions,eventdev_selftest_sw, > member_perf,efd_perf,lpm6_perf,red_perf,distributor_perf > > Currently after the above blacklist update, the total execution time of "make test-fast" is 54s. Less than a minute is a good limit! Maybe we can do even better with above suggestions. Thanks for working on it. Feel free to send a new patch when you think it is ready. We can try to have the new autotests when starting 18.08. Then we should communicate to enforce a better coverage of the autotests.
1/4: new test cases are added to autotest config 2/4: skipped test case result is updated appropriately 3/4: test cmdline is updated to support freebsd 4/4: added make targets for group of testcases --- v4: classify testcases based on priority and run with different targets --- Jananee Parthasarathy (4): app/test: enhance autotest config app/test: update result for skipped test cases app/test: enhance freebsd support in autotest config mk: update make targets for classified testcases mk/rte.sdkroot.mk | 4 +- mk/rte.sdktest.mk | 33 ++- test/test/autotest_data.py | 471 ++++++++++++++++++++++++++++++++++++++- test/test/autotest_runner.py | 8 +- test/test/autotest_test_funcs.py | 6 +- 5 files changed, 502 insertions(+), 20 deletions(-) mode change 100644 => 100755 test/test/autotest_data.py mode change 100644 => 100755 test/test/autotest_test_funcs.py -- 1.7.12.2
Autotest config is enhanced with additional test cases being added to autotest_data.py Signed-off-by: Jananee Parthasarathy <jananeex.m.parthasarathy@intel.com> Reviewed-by: Reshma Pattan <reshma.pattan@intel.com> Reviewed-by: Anatoly Burakov <anatoly.burakov@intel.com> --- test/test/autotest_data.py | 471 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 462 insertions(+), 9 deletions(-) mode change 100644 => 100755 test/test/autotest_data.py diff --git a/test/test/autotest_data.py b/test/test/autotest_data.py old mode 100644 new mode 100755 index aacfe0a..2c6828d --- a/test/test/autotest_data.py +++ b/test/test/autotest_data.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2010-2018 Intel Corporation # Test data for autotests @@ -70,6 +70,31 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Eventdev selftest octeontx", + "Command": "eventdev_selftest_octeontx", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Event ring autotest", + "Command": "event_ring_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Table autotest", + "Command": "table_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Flow classify autotest", + "Command": "flow_classify_autotest", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -125,6 +150,50 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Dump struct sizes", + "Command": "dump_struct_sizes", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump mempool", + "Command": "dump_mempool", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump malloc stats", + "Command": "dump_malloc_stats", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump devargs", + "Command": "dump_devargs", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump log types", + "Command": "dump_log_types", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump_ring", + "Command": "dump_ring", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Quit", + "Command": "quit", + "Func": default_autotest, + "Report": None, + }, + + ], }, { @@ -168,6 +237,26 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Set rxtx mode", + "Command": "set_rxtx_mode", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Set rxtx anchor", + "Command": "set_rxtx_anchor", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Set rxtx sc", + "Command": "set_rxtx_sc", + "Func": default_autotest, + "Report": None, + }, + + ] }, { @@ -176,12 +265,6 @@ def per_sockets(num): "Tests": [ { - "Name": "PCI autotest", - "Command": "pci_autotest", - "Func": default_autotest, - "Report": None, - }, - { "Name": "Malloc autotest", "Command": "malloc_autotest", "Func": default_autotest, @@ -211,6 +294,37 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Event eth rx adapter autotest", + "Command": "event_eth_rx_adapter_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Rawdev autotest", + "Command": "rawdev_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Kvargs autotest", + "Command": "kvargs_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump physmem", + "Command": "dump_physmem", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Dump memzone", + "Command": "dump_memzone", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -248,6 +362,31 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Devargs autotest", + "Command": "devargs_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Link bonding autotest", + "Command": "link_bonding_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Link bonding mode4 autotest", + "Command": "link_bonding_mode4_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Link bonding rssconf autotest", + "Command": "link_bonding_rssconf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -285,6 +424,25 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Crc autotest", + "Command": "crc_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Distributor autotest", + "Command": "distributor_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Reorder autotest", + "Command": "reorder_autotest", + "Func": default_autotest, + "Report": None, + }, + ] }, { @@ -310,8 +468,184 @@ def per_sockets(num): "Func": default_autotest, "Report": None, }, + { + "Name": "Barrier autotest", + "Command": "barrier_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "User delay", + "Command": "user_delay_us", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Bitmap test", + "Command": "bitmap_test", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Hash scaling autotest", + "Command": "hash_scaling_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Hash multiwriter autotest", + "Command": "hash_multiwriter_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Service autotest", + "Command": "service_autotest", + "Func": default_autotest, + "Report": None, + }, + + + ] + }, + { + "Prefix": "group_8", + "Memory": "128", + "Tests": + [ + { + "Name": "Timer racecond autotest", + "Command": "timer_racecond_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Member autotest", + "Command": "member_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Efd_autotest", + "Command": "efd_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Thash autotest", + "Command": "thash_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Hash function autotest", + "Command": "hash_functions_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "group_9", + "Memory": "64", + "Tests": + [ + { + "Name": "Cryptodev sw mrvl autotest", + "Command": "cryptodev_sw_mrvl_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev dpaa2 sec autotest", + "Command": "cryptodev_dpaa2_sec_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev dpaa sec autotest", + "Command": "cryptodev_dpaa_sec_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev qat autotest", + "Command": "cryptodev_qat_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev aesni mb autotest", + "Command": "cryptodev_aesni_mb_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev openssl autotest", + "Command": "cryptodev_openssl_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev scheduler autotest", + "Command": "cryptodev_scheduler_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev aesni gcm autotest", + "Command": "cryptodev_aesni_gcm_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev null autotest", + "Command": "cryptodev_null_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev sw snow3g autotest", + "Command": "cryptodev_sw_snow3g_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev sw kasumi autotest", + "Command": "cryptodev_sw_kasumi_autotest", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Cryptodev_sw_zuc_autotest", + "Command": "cryptodev_sw_zuc_autotest", + "Func": default_autotest, + "Report": None, + }, + + ] }, + { + "Prefix": "group_10", + "Memory": "64", + "Tests": + [ + { + "Name": "Reciprocal division", + "Command": "reciprocal_division", + "Func": default_autotest, + "Report": None, + }, + { + "Name": "Red all", + "Command": "red_all", + "Func": default_autotest, + "Report": None, + }, + ] + }, + ] # tests that should not be run when any other tests are running @@ -336,8 +670,8 @@ def per_sockets(num): "Tests": [ { - "Name": "Eventdev sw autotest", - "Command": "eventdev_sw_autotest", + "Name": "Eventdev selftest sw", + "Command": "eventdev_selftest_sw", "Func": default_autotest, "Report": None, }, @@ -447,6 +781,125 @@ def per_sockets(num): }, ] }, + { + "Prefix": "reciprocal_division_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Reciprocal division perf", + "Command": "reciprocal_division_perf", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "member_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Member perf autotest", + "Command": "member_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "efd_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Efd perf autotest", + "Command": "efd_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "lpm_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Lpm perf autotest", + "Command": "lpm_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "lpm6_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Lpm6 perf autotest", + "Command": "lpm6_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "red_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Red_perf", + "Command": "red_perf", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "distributor_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Distributor perf autotest", + "Command": "distributor_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "ring_pmd_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Ring pmd perf autotest", + "Command": "ring_pmd_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + { + "Prefix": "pmd_perf", + "Memory": per_sockets(256), + "Tests": + [ + { + "Name": "Pmd perf autotest", + "Command": "pmd_perf_autotest", + "Func": default_autotest, + "Report": None, + }, + ] + }, + + # # Please always make sure that ring_perf is the last test! -- 1.7.12.2
Fixed in autotest_test_funcs.py to handle test cases which returns "Skipped" as result. The issue was skipped test cases got timed out, causing delay in autotests execution. Signed-off-by: Jananee Parthasarathy <jananeex.m.parthasarathy@intel.com> Reviewed-by: Reshma Pattan <reshma.pattan@intel.com> Reviewed-by: Anatoly Burakov <anatoly.burakov@intel.com> --- test/test/autotest_test_funcs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) mode change 100644 => 100755 test/test/autotest_test_funcs.py diff --git a/test/test/autotest_test_funcs.py b/test/test/autotest_test_funcs.py old mode 100644 new mode 100755 index 65fe335..219c208 --- a/test/test/autotest_test_funcs.py +++ b/test/test/autotest_test_funcs.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2010-2018 Intel Corporation # Test functions @@ -12,12 +12,14 @@ def default_autotest(child, test_name): child.sendline(test_name) result = child.expect(["Test OK", "Test Failed", - "Command not found", pexpect.TIMEOUT], timeout=900) + "Command not found", "Skipped", pexpect.TIMEOUT], timeout=900) if result == 1: return -1, "Fail" elif result == 2: return -1, "Fail [Not found]" elif result == 3: + return -1, "Fail [Test returns Skipped]" + elif result == 4: return -1, "Fail [Timeout]" return 0, "Success" -- 1.7.12.2
Update test command line with no additional arguments, as FreeBSD doesn't support socket-mem, file-prefix options Signed-off-by: Jananee Parthasarathy <jananeex.m.parthasarathy@intel.com> Reviewed-by: Reshma Pattan <reshma.pattan@intel.com> Reviewed-by: Anatoly Burakov <anatoly.burakov@intel.com> --- test/test/autotest_runner.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/test/autotest_runner.py b/test/test/autotest_runner.py index a692f06..0997e78 100644 --- a/test/test/autotest_runner.py +++ b/test/test/autotest_runner.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2010-2018 Intel Corporation # The main logic behind running autotests in parallel @@ -201,6 +201,12 @@ def __init__(self, cmdline, target, blacklist, whitelist): # set up cmdline string def __get_cmdline(self, test): cmdline = self.cmdline + os_name = sys.platform + + # test command line need not be modified for FreeBSD + # as socket-mem, file-prefix is not supported + if os_name.startswith('freebsd') : + return cmdline # append memory limitations for each test # otherwise tests won't run in parallel -- 1.7.12.2
Makefiles are updated with new test case lists. Test cases are classified as - P1 - Main test cases, P2 - Cryptodev/driver test cases, P3 - Perf test cases which takes longer than 10s, P4 - Logging/Dump test cases. Makefile is updated with different targets for the above classified groups. Test cases for different targets are listed accordingly. Signed-off-by: Jananee Parthasarathy <jananeex.m.parthasarathy@intel.com> Reviewed-by: Reshma Pattan <reshma.pattan@intel.com> --- mk/rte.sdkroot.mk | 4 ++-- mk/rte.sdktest.mk | 33 +++++++++++++++++++++++++++------ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/mk/rte.sdkroot.mk b/mk/rte.sdkroot.mk index f43cc78..ea3473e 100644 --- a/mk/rte.sdkroot.mk +++ b/mk/rte.sdkroot.mk @@ -68,8 +68,8 @@ config defconfig showconfigs showversion showversionum: cscope gtags tags etags: $(Q)$(RTE_SDK)/devtools/build-tags.sh $@ $T -.PHONY: test test-basic test-fast test-ring test-mempool test-perf coverage -test test-basic test-fast test-ring test-mempool test-perf coverage: +.PHONY: test test-basic test-fast test-ring test-mempool test-perf coverage test-drivers test-dump +test test-basic test-fast test-ring test-mempool test-perf coverage test-drivers test-dump: $(Q)$(MAKE) -f $(RTE_SDK)/mk/rte.sdktest.mk $@ test: test-build diff --git a/mk/rte.sdktest.mk b/mk/rte.sdktest.mk index ee1fe0c..13d1efb 100644 --- a/mk/rte.sdktest.mk +++ b/mk/rte.sdktest.mk @@ -18,14 +18,35 @@ DIR := $(shell basename $(RTE_OUTPUT)) # # test: launch auto-tests, very simple for now. # -.PHONY: test test-basic test-fast test-perf coverage +.PHONY: test test-basic test-fast test-perf test-drivers test-dump coverage -PERFLIST=ring_perf,mempool_perf,memcpy_perf,hash_perf,timer_perf -coverage: BLACKLIST=-$(PERFLIST) -test-fast: BLACKLIST=-$(PERFLIST) -test-perf: WHITELIST=$(PERFLIST) +PERFLIST=ring_perf,mempool_perf,memcpy_perf,hash_perf,timer_perf,\ + reciprocal_division,reciprocal_division_perf,lpm_perf,red_all,\ + barrier,hash_multiwriter,timer_racecond,efd,hash_functions,\ + eventdev_selftest_sw,member_perf,efd_perf,lpm6_perf,red_perf,\ + distributor_perf,ring_pmd_perf,pmd_perf,ring_perf +DRIVERSLIST=link_bonding,link_bonding_mode4,link_bonding_rssconf,\ + cryptodev_sw_mrvl,cryptodev_dpaa2_sec,cryptodev_dpaa_sec,\ + cryptodev_qat,cryptodev_aesni_mb,cryptodev_openssl,\ + cryptodev_scheduler,cryptodev_aesni_gcm,cryptodev_null,\ + cryptodev_sw_snow3g,cryptodev_sw_kasumi,cryptodev_sw_zuc +DUMPLIST=dump_struct_sizes,dump_mempool,dump_malloc_stats,dump_devargs,\ + dump_log_types,dump_ring,quit,dump_physmem,dump_memzone,\ + devargs_autotest -test test-basic test-fast test-perf: +SPACESTR:= +SPACESTR+= +STRIPPED_PERFLIST=$(subst $(SPACESTR),,$(PERFLIST)) +STRIPPED_DRIVERSLIST=$(subst $(SPACESTR),,$(DRIVERSLIST)) +STRIPPED_DUMPLIST=$(subst $(SPACESTR),,$(DUMPLIST)) + +coverage: BLACKLIST=-$(STRIPPED_PERFLIST) +test-fast: BLACKLIST=-$(STRIPPED_PERFLIST),$(STRIPPED_DRIVERSLIST),$(STRIPPED_DUMPLIST) +test-perf: WHITELIST=$(STRIPPED_PERFLIST) +test-drivers: WHITELIST=$(STRIPPED_DRIVERSLIST) +test-dump: WHITELIST=$(STRIPPED_DUMPLIST) + +test test-basic test-fast test-perf test-drivers test-dump: @mkdir -p $(AUTOTEST_DIR) ; \ cd $(AUTOTEST_DIR) ; \ if [ -f $(RTE_OUTPUT)/app/test ]; then \ -- 1.7.12.2