[PATCH v6 6/9] dts: add autodoc pydantic

[Luca Vizzarro <luca.vizzarro@arm.com>]

Add and enable the autodoc-pydantic sphinx extension. Pydantic models
are not correctly recognised by autodoc, causing the generated docs to
lack all the actual model information. The autodoc-pydantic sphinx
extension fixes the original behaviour by correctly formatting them.

Signed-off-by: Luca Vizzarro <luca.vizzarro@arm.com>
Reviewed-by: Paul Szczepanek <paul.szczepanek@arm.com>
Reviewed-by: Nicholas Pratte <npratte@iol.unh.edu>
Reviewed-by: Patrick Robb <probb@iol.unh.edu>
---
 doc/guides/conf.py       |  13 +++
 doc/guides/tools/dts.rst | 187 ++-------------------------------------
 dts/poetry.lock          |  59 +++++++++++-
 dts/pyproject.toml       |   1 +
 4 files changed, 79 insertions(+), 181 deletions(-)

diff --git a/doc/guides/conf.py b/doc/guides/conf.py
index b553d9d5bf..71fed45b3d 100644
--- a/doc/guides/conf.py
+++ b/doc/guides/conf.py
@@ -60,6 +60,19 @@
 # DTS API docs additional configuration
 if environ.get('DTS_DOC_BUILD'):
     extensions = ['sphinx.ext.napoleon', 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx']
+
+    # Pydantic models require autodoc_pydantic for the right formatting
+    try:
+        import sphinxcontrib.autodoc_pydantic
+
+        extensions.append("sphinxcontrib.autodoc_pydantic")
+    except ImportError:
+        print(
+            "The DTS API doc dependencies are missing. The generated output won't be "
+            "as intended, and autodoc may throw unexpected warnings.",
+            file=stderr,
+        )
+
     # Napoleon enables the Google format of Python doscstrings.
     napoleon_numpy_docstring = False
     napoleon_attr_annotations = True
diff --git a/doc/guides/tools/dts.rst b/doc/guides/tools/dts.rst
index c52de1808c..fb6504fa59 100644
--- a/doc/guides/tools/dts.rst
+++ b/doc/guides/tools/dts.rst
@@ -204,9 +204,10 @@ node, and then run the tests with the newly built binaries.
 Configuring DTS
 ~~~~~~~~~~~~~~~
 
-DTS configuration is split into nodes and test runs and build targets within test runs,
-and follows a defined schema as described in `Configuration Schema`_.
-By default, DTS will try to use the ``dts/conf.yaml`` :ref:`config file <configuration_schema_example>`,
+DTS configuration is split into nodes and test runs, and must respect the model definitions as
+documented in the DTS API docs under the ``config`` page. The root of the configuration is
+represented by the ``Configuration`` model.
+By default, DTS will try to use the ``dts/conf.yaml`` :ref:`config file <configuration_example>`,
 which is a template that illustrates what can be configured in DTS.
 
 The user must have :ref:`administrator privileges <sut_admin_user>`
@@ -470,184 +471,10 @@ The output is generated in ``build/doc/api/dts/html``.
 
    Make sure to fix any Sphinx warnings when adding or updating docstrings.
 
+.. _configuration_example:
 
-Configuration Schema
---------------------
-
-Definitions
-~~~~~~~~~~~
-
-_`Node name`
-   *string* – A unique identifier for a node.
-   **Examples**: ``SUT1``, ``TG1``.
-
-_`ARCH`
-   *string* – The CPU architecture.
-   **Supported values**: ``x86_64``, ``arm64``, ``ppc64le``.
-
-_`CPU`
-   *string* – The CPU microarchitecture. Use ``native`` for x86.
-   **Supported values**: ``native``, ``armv8a``, ``dpaa2``, ``thunderx``, ``xgene1``.
-
-_`OS`
-   *string* – The operating system. **Supported values**: ``linux``.
-
-_`Compiler`
-   *string* – The compiler used for building DPDK.
-   **Supported values**: ``gcc``, ``clang``, ``icc``, ``mscv``.
-
-_`Build target`
-   *mapping* – Build targets supported by DTS for building DPDK, described as:
-
-   ==================== =================================================================
-   ``arch``             See `ARCH`_
-   ``os``               See `OS`_
-   ``cpu``              See `CPU`_
-   ``compiler``         See `Compiler`_
-   ``compiler_wrapper`` *string* – Value prepended to the CC variable for the DPDK build.
-
-                        **Example**: ``ccache``
-   ==================== =================================================================
-
-_`hugepages_2mb`
-   *mapping* – hugepages_2mb described as:
-
-   ==================== ================================================================
-   ``number_of``        *integer* – The number of 2MB hugepages to configure.
-
-                        Hugepage size will be the system default.
-   ``force_first_numa`` (*optional*, defaults to ``false``) – If ``true``, it forces the
-
-                        configuration of hugepages on the first NUMA node.
-   ==================== ================================================================
-
-_`Network port`
-   *mapping* – the NIC port described as:
-
-   ====================== =================================================================================
-   ``pci``                *string* – the local PCI address of the port. **Example**: ``0000:00:08.0``
-   ``os_driver_for_dpdk`` | *string* – this port's device driver when using with DPDK
-                          | When setting up the SUT, DTS will bind the network device to this driver
-                          | for compatibility with DPDK.
-
-                          **Examples**: ``vfio-pci``, ``mlx5_core``
-   ``os_driver``          | *string* – this port's device driver when **not** using with DPDK
-                          | When tearing down the tests on the SUT, DTS will bind the network device
-                          | *back* to this driver. This driver is meant to be the one that the SUT would
-                          | normally use for this device, or whichever driver it is preferred to leave the
-                          | device bound to after testing.
-                          | This also represents the driver that is used in conjunction with the traffic
-                          | generator software.
-
-                          **Examples**: ``i40e``, ``mlx5_core``
-   ``peer_node``          *string* – the name of the peer node connected to this port.
-   ``peer_pci``           *string* – the PCI address of the peer node port. **Example**: ``000a:01:00.1``
-   ====================== =================================================================================
-
-_`Test suite`
-   *string* – name of the test suite to run. **Examples**: ``hello_world``, ``os_udp``
-
-_`Test target`
-   *mapping* – selects specific test cases to run from a test suite. Mapping is described as follows:
-
-   ========= ===============================================================================================
-   ``suite`` See `Test suite`_
-   ``cases`` (*optional*) *sequence* of *string* – list of the selected test cases in the test suite to run.
-
-             Unknown test cases will be silently ignored.
-   ========= ===============================================================================================
-
-
-Properties
-~~~~~~~~~~
-
-The configuration requires listing all the test run environments and nodes
-involved in the testing. These can be defined with the following mappings:
-
-``test runs``
-   `sequence <https://docs.python.org/3/library/stdtypes.html#sequence-types-list-tuple-range>`_ listing
-   the test run environments. Each entry is described as per the following
-   `mapping <https://docs.python.org/3/library/stdtypes.html#mapping-types-dict>`_:
-
-   +----------------------------+-------------------------------------------------------------------+
-   | ``build_targets``          | *sequence* of `Build target`_                                     |
-   +----------------------------+-------------------------------------------------------------------+
-   | ``perf``                   | *boolean* – Enable performance testing.                           |
-   +----------------------------+-------------------------------------------------------------------+
-   | ``func``                   | *boolean* – Enable functional testing.                            |
-   +----------------------------+-------------------------------------------------------------------+
-   | ``test_suites``            | *sequence* of **one of** `Test suite`_ **or** `Test target`_      |
-   +----------------------------+-------------------------------------------------------------------+
-   | ``skip_smoke_tests``       | (*optional*) *boolean* – Allows you to skip smoke testing         |
-   |                            | if ``true``.                                                      |
-   +----------------------------+-------------------------------------------------------------------+
-   | ``system_under_test_node`` | System under test node specified with:                            |
-   |                            +---------------+---------------------------------------------------+
-   |                            | ``node_name`` | See `Node name`_                                  |
-   |                            +---------------+---------------------------------------------------+
-   |                            | ``vdevs``     | (*optional*) *sequence* of *string*               |
-   |                            |               |                                                   |
-   |                            |               | List of virtual devices passed with the ``--vdev``|
-   |                            |               | argument to DPDK. **Example**: ``crypto_openssl`` |
-   +----------------------------+---------------+---------------------------------------------------+
-   | ``traffic_generator_node`` | Node name for the traffic generator node.                         |
-   +----------------------------+-------------------------------------------------------------------+
-   | ``random_seed``            | (*optional*) *int* – Set a seed for pseudo-random generation.     |
-   +----------------------------+-------------------------------------------------------------------+
-
-``nodes``
-   `sequence <https://docs.python.org/3/library/stdtypes.html#sequence-types-list-tuple-range>`_ listing
-   the nodes. Each entry is described as per the following
-   `mapping <https://docs.python.org/3/library/stdtypes.html#mapping-types-dict>`_:
-
-   +-----------------------+---------------------------------------------------------------------------------------+
-   | ``name``              | See `Node name`_                                                                      |
-   +-----------------------+---------------------------------------------------------------------------------------+
-   | ``hostname``          | *string* – The network hostname or IP address of this node.                           |
-   +-----------------------+---------------------------------------------------------------------------------------+
-   | ``user``              | *string* – The SSH user credential to use to login to this node.                      |
-   +-----------------------+---------------------------------------------------------------------------------------+
-   | ``password``          | (*optional*) *string* – The SSH password credential for this node.                    |
-   |                       |                                                                                       |
-   |                       | **NB**: Use only as last resort. SSH keys are **strongly** preferred.                 |
-   +-----------------------+---------------------------------------------------------------------------------------+
-   | ``arch``              | The architecture of this node. See `ARCH`_ for supported values.                      |
-   +-----------------------+---------------------------------------------------------------------------------------+
-   | ``os``                | The operating system of this node. See `OS`_ for supported values.                    |
-   +-----------------------+---------------------------------------------------------------------------------------+
-   | ``lcores``            | | (*optional*, defaults to 1) *string* – Comma-separated list of logical              |
-   |                       | | cores to use. An empty string means use all lcores.                                 |
-   |                       |                                                                                       |
-   |                       | **Example**: ``1,2,3,4,5,18-22``                                                      |
-   +-----------------------+---------------------------------------------------------------------------------------+
-   | ``use_first_core``    | (*optional*, defaults to ``false``) *boolean*                                         |
-   |                       |                                                                                       |
-   |                       | Indicates whether DPDK should use only the first physical core or not.                |
-   +-----------------------+---------------------------------------------------------------------------------------+
-   | ``memory_channels``   | (*optional*, defaults to 1) *integer*                                                 |
-   |                       |                                                                                       |
-   |                       | The number of the memory channels to use.                                             |
-   +-----------------------+---------------------------------------------------------------------------------------+
-   | ``hugepages_2mb``     | (*optional*) See `hugepages_2mb`_. If unset, 2MB hugepages won't be configured        |
-   |                       |                                                                                       |
-   |                       | in favour of the system configuration.                                                |
-   +-----------------------+---------------------------------------------------------------------------------------+
-   | ``ports``             | | *sequence* of `Network port`_ – Describe ports that are **directly** paired with    |
-   |                       | | other nodes used in conjunction with this one. Both ends of the links must be       |
-   |                       | | described. If there any inconsistencies DTS won't run.                              |
-   |                       |                                                                                       |
-   |                       | **Example**: port 1 of node ``SUT1`` is connected to port 1 of node ``TG1`` etc.      |
-   +-----------------------+---------------------------------------------------------------------------------------+
-   | ``traffic_generator`` | (*optional*) Traffic generator, if any, setup on this node described as:              |
-   |                       +----------+----------------------------------------------------------------------------+
-   |                       | ``type`` | *string* – **Supported values**: *SCAPY*                                   |
-   +-----------------------+----------+----------------------------------------------------------------------------+
-
-
-.. _configuration_schema_example:
-
-Example
-~~~~~~~
+Configuration Example
+---------------------
 
 The following example (which can be found in ``dts/conf.yaml``) sets up two nodes:
 
diff --git a/dts/poetry.lock b/dts/poetry.lock
index 9f7db60793..ee564676b4 100644
--- a/dts/poetry.lock
+++ b/dts/poetry.lock
@@ -34,6 +34,29 @@ files = [
     {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"},
 ]
 
+[[package]]
+name = "autodoc-pydantic"
+version = "2.2.0"
+description = "Seamlessly integrate pydantic models in your Sphinx documentation."
+optional = false
+python-versions = "<4.0.0,>=3.8.1"
+files = [
+    {file = "autodoc_pydantic-2.2.0-py3-none-any.whl", hash = "sha256:8c6a36fbf6ed2700ea9c6d21ea76ad541b621fbdf16b5a80ee04673548af4d95"},
+]
+
+[package.dependencies]
+pydantic = ">=2.0,<3.0.0"
+pydantic-settings = ">=2.0,<3.0.0"
+Sphinx = ">=4.0"
+
+[package.extras]
+docs = ["myst-parser (>=3.0.0,<4.0.0)", "sphinx-copybutton (>=0.5.0,<0.6.0)", "sphinx-rtd-theme (>=2.0.0,<3.0.0)", "sphinx-tabs (>=3,<4)", "sphinxcontrib-mermaid (>=0.9.0,<0.10.0)"]
+erdantic = ["erdantic (<2.0)"]
+linting = ["ruff (>=0.4.0,<0.5.0)"]
+security = ["pip-audit (>=2.7.2,<3.0.0)"]
+test = ["coverage (>=7,<8)", "defusedxml (>=0.7.1)", "pytest (>=8.0.0,<9.0.0)", "pytest-sugar (>=1.0.0,<2.0.0)"]
+type-checking = ["mypy (>=1.9,<2.0)", "types-docutils (>=0.20,<0.21)", "typing-extensions (>=4.11,<5.0)"]
+
 [[package]]
 name = "babel"
 version = "2.13.1"
@@ -829,6 +852,26 @@ files = [
 [package.dependencies]
 typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
 
+[[package]]
+name = "pydantic-settings"
+version = "2.6.0"
+description = "Settings management using Pydantic"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pydantic_settings-2.6.0-py3-none-any.whl", hash = "sha256:4a819166f119b74d7f8c765196b165f95cc7487ce58ea27dec8a5a26be0970e0"},
+    {file = "pydantic_settings-2.6.0.tar.gz", hash = "sha256:44a1804abffac9e6a30372bb45f6cafab945ef5af25e66b1c634c01dd39e0188"},
+]
+
+[package.dependencies]
+pydantic = ">=2.7.0"
+python-dotenv = ">=0.21.0"
+
+[package.extras]
+azure-key-vault = ["azure-identity (>=1.16.0)", "azure-keyvault-secrets (>=4.8.0)"]
+toml = ["tomli (>=2.0.1)"]
+yaml = ["pyyaml (>=6.0.1)"]
+
 [[package]]
 name = "pydocstyle"
 version = "6.1.1"
@@ -935,6 +978,20 @@ cffi = ">=1.4.1"
 docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"]
 tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"]
 
+[[package]]
+name = "python-dotenv"
+version = "1.0.1"
+description = "Read key-value pairs from a .env file and set them as environment variables"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"},
+    {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"},
+]
+
+[package.extras]
+cli = ["click (>=5.0)"]
+
 [[package]]
 name = "pyyaml"
 version = "6.0.1"
@@ -1304,4 +1361,4 @@ zstd = ["zstandard (>=0.18.0)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "310e2d3725e20ffc6ef017db92e8000c042eb2ac98a1a5eb441de17c87417e9f"
+content-hash = "fe9a9fdf7b43e8dce2fb5ee600921d4047fef2f4037a78bbd150f71df202493e"
diff --git a/dts/pyproject.toml b/dts/pyproject.toml
index 9a3fb02ee9..f69c70877a 100644
--- a/dts/pyproject.toml
+++ b/dts/pyproject.toml
@@ -44,6 +44,7 @@ optional = true
 sphinx = "<=7"
 sphinx-rtd-theme = ">=1.2.2"
 pyelftools = "^0.31"
+autodoc-pydantic = "^2.2.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
-- 
2.43.0