Re: [PATCH v3 3/3] dts: add API doc generation

[Jeremy Spewock <jspewock@iol.unh.edu>]

On Mon, Jan 22, 2024 at 11:35 AM Juraj Linkeš
<juraj.linkes@pantheon.tech> wrote:
>
> The tool used to generate developer docs is Sphinx, which is already in
> use in DPDK. The same configuration is used to preserve style, but it's
> been augmented with doc-generating configuration. There's a change that
> modifies how the sidebar displays the content hierarchy that's been put
> into an if block to not interfere with regular docs.
>
> Sphinx generates the documentation from Python docstrings. The docstring
> format is the Google format [0] which requires the sphinx.ext.napoleon
> extension. The other extension, sphinx.ext.intersphinx, enables linking
> to object in external documentations, such as the Python documentation.
>
> There are two requirements for building DTS docs:
> * The same Python version as DTS or higher, because Sphinx imports the
>   code.
> * Also the same Python packages as DTS, for the same reason.
>
> [0] https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings
>
> Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
> ---
>  buildtools/call-sphinx-build.py | 33 +++++++++++++++++++---------
>  doc/api/doxy-api-index.md       |  3 +++
>  doc/api/doxy-api.conf.in        |  2 ++
>  doc/api/meson.build             | 11 +++++++---
>  doc/guides/conf.py              | 39 ++++++++++++++++++++++++++++-----
>  doc/guides/meson.build          |  1 +
>  doc/guides/tools/dts.rst        | 34 +++++++++++++++++++++++++++-
>  dts/doc/meson.build             | 27 +++++++++++++++++++++++
>  dts/meson.build                 | 16 ++++++++++++++
>  meson.build                     |  1 +
>  10 files changed, 148 insertions(+), 19 deletions(-)
>  create mode 100644 dts/doc/meson.build
>  create mode 100644 dts/meson.build
>
> diff --git a/buildtools/call-sphinx-build.py b/buildtools/call-sphinx-build.py
> index 39a60d09fa..aea771a64e 100755
> --- a/buildtools/call-sphinx-build.py
> +++ b/buildtools/call-sphinx-build.py
> @@ -3,37 +3,50 @@
>  # Copyright(c) 2019 Intel Corporation
>  #
>
> +import argparse
>  import sys
>  import os
>  from os.path import join
>  from subprocess import run, PIPE, STDOUT
>  from packaging.version import Version
>
> -# assign parameters to variables
> -(sphinx, version, src, dst, *extra_args) = sys.argv[1:]
> +parser = argparse.ArgumentParser()
> +parser.add_argument('sphinx')
> +parser.add_argument('version')
> +parser.add_argument('src')
> +parser.add_argument('dst')
> +parser.add_argument('--dts-root', default=None)
> +args, extra_args = parser.parse_known_args()
>
>  # set the version in environment for sphinx to pick up
> -os.environ['DPDK_VERSION'] = version
> +os.environ['DPDK_VERSION'] = args.version
> +if args.dts_root:
> +    os.environ['DTS_ROOT'] = args.dts_root
>
>  # for sphinx version >= 1.7 add parallelism using "-j auto"
> -ver = run([sphinx, '--version'], stdout=PIPE,
> +ver = run([args.sphinx, '--version'], stdout=PIPE,
>            stderr=STDOUT).stdout.decode().split()[-1]
> -sphinx_cmd = [sphinx] + extra_args
> +sphinx_cmd = [args.sphinx] + extra_args
>  if Version(ver) >= Version('1.7'):
>      sphinx_cmd += ['-j', 'auto']
>
>  # find all the files sphinx will process so we can write them as dependencies
>  srcfiles = []
> -for root, dirs, files in os.walk(src):
> +for root, dirs, files in os.walk(args.src):
>      srcfiles.extend([join(root, f) for f in files])
>
> +if not os.path.exists(args.dst):
> +    os.makedirs(args.dst)
> +
>  # run sphinx, putting the html output in a "html" directory
> -with open(join(dst, 'sphinx_html.out'), 'w') as out:
> -    process = run(sphinx_cmd + ['-b', 'html', src, join(dst, 'html')],
> -                  stdout=out)
> +with open(join(args.dst, 'sphinx_html.out'), 'w') as out:
> +    process = run(
> +        sphinx_cmd + ['-b', 'html', args.src, join(args.dst, 'html')],
> +        stdout=out
> +    )
>
>  # create a gcc format .d file giving all the dependencies of this doc build
> -with open(join(dst, '.html.d'), 'w') as d:
> +with open(join(args.dst, '.html.d'), 'w') as d:
>      d.write('html: ' + ' '.join(srcfiles) + '\n')
>
>  sys.exit(process.returncode)
> diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
> index a6a768bd7c..b49b24acce 100644
> --- a/doc/api/doxy-api-index.md
> +++ b/doc/api/doxy-api-index.md
> @@ -241,3 +241,6 @@ The public API headers are grouped by topics:
>    [experimental APIs](@ref rte_compat.h),
>    [ABI versioning](@ref rte_function_versioning.h),
>    [version](@ref rte_version.h)
> +
> +- **tests**:
> +  [**DTS**](@dts_api_main_page)
> diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
> index e94c9e4e46..d53edeba57 100644
> --- a/doc/api/doxy-api.conf.in
> +++ b/doc/api/doxy-api.conf.in
> @@ -121,6 +121,8 @@ SEARCHENGINE            = YES
>  SORT_MEMBER_DOCS        = NO
>  SOURCE_BROWSER          = YES
>
> +ALIASES                 = "dts_api_main_page=@DTS_API_MAIN_PAGE@"
> +
>  EXAMPLE_PATH            = @TOPDIR@/examples
>  EXAMPLE_PATTERNS        = *.c
>  EXAMPLE_RECURSIVE       = YES
> diff --git a/doc/api/meson.build b/doc/api/meson.build
> index 5b50692df9..ffc75d7b5a 100644
> --- a/doc/api/meson.build
> +++ b/doc/api/meson.build
> @@ -1,6 +1,7 @@
>  # SPDX-License-Identifier: BSD-3-Clause
>  # Copyright(c) 2018 Luca Boccassi <bluca@debian.org>
>
> +doc_api_build_dir = meson.current_build_dir()
>  doxygen = find_program('doxygen', required: get_option('enable_docs'))
>
>  if not doxygen.found()
> @@ -32,14 +33,18 @@ example = custom_target('examples.dox',
>  # set up common Doxygen configuration
>  cdata = configuration_data()
>  cdata.set('VERSION', meson.project_version())
> -cdata.set('API_EXAMPLES', join_paths(dpdk_build_root, 'doc', 'api', 'examples.dox'))
> -cdata.set('OUTPUT', join_paths(dpdk_build_root, 'doc', 'api'))
> +cdata.set('API_EXAMPLES', join_paths(doc_api_build_dir, 'examples.dox'))
> +cdata.set('OUTPUT', doc_api_build_dir)
>  cdata.set('TOPDIR', dpdk_source_root)
> -cdata.set('STRIP_FROM_PATH', ' '.join([dpdk_source_root, join_paths(dpdk_build_root, 'doc', 'api')]))
> +cdata.set('STRIP_FROM_PATH', ' '.join([dpdk_source_root, doc_api_build_dir]))
>  cdata.set('WARN_AS_ERROR', 'NO')
>  if get_option('werror')
>      cdata.set('WARN_AS_ERROR', 'YES')
>  endif
> +# A local reference must be relative to the main index.html page
> +# The path below can't be taken from the DTS meson file as that would
> +# require recursive subdir traversal (doc, dts, then doc again)
> +cdata.set('DTS_API_MAIN_PAGE', join_paths('..', 'dts', 'html', 'index.html'))
>
>  # configure HTML Doxygen run
>  html_cdata = configuration_data()
> diff --git a/doc/guides/conf.py b/doc/guides/conf.py
> index 0f7ff5282d..b442a1f76c 100644
> --- a/doc/guides/conf.py
> +++ b/doc/guides/conf.py
> @@ -7,10 +7,9 @@
>  from sphinx import __version__ as sphinx_version
>  from os import listdir
>  from os import environ
> -from os.path import basename
> -from os.path import dirname
> +from os.path import basename, dirname
>  from os.path import join as path_join
> -from sys import argv, stderr
> +from sys import argv, stderr, path
>
>  import configparser
>
> @@ -24,6 +23,37 @@
>            file=stderr)
>      pass
>
> +# Napoleon enables the Google format of Python doscstrings, used in DTS
> +# Intersphinx allows linking to external projects, such as Python docs, also used in DTS
> +extensions = ['sphinx.ext.napoleon', 'sphinx.ext.intersphinx']
> +
> +# DTS Python docstring options
> +autodoc_default_options = {
> +    'members': True,
> +    'member-order': 'bysource',
> +    'show-inheritance': True,
> +}
> +autodoc_class_signature = 'separated'
> +autodoc_typehints = 'both'
> +autodoc_typehints_format = 'short'
> +autodoc_typehints_description_target = 'documented'
> +napoleon_numpy_docstring = False
> +napoleon_attr_annotations = True
> +napoleon_preprocess_types = True
> +add_module_names = False
> +toc_object_entries = True
> +toc_object_entries_show_parents = 'hide'
> +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)}
> +
> +dts_root = environ.get('DTS_ROOT')
> +if dts_root:
> +    path.append(dts_root)
> +    # DTS Sidebar config
> +    html_theme_options = {
> +        'collapse_navigation': False,
> +        'navigation_depth': -1,
> +    }
> +
>  stop_on_error = ('-W' in argv)
>
>  project = 'Data Plane Development Kit'
> @@ -35,8 +65,7 @@
>  html_show_copyright = False
>  highlight_language = 'none'
>
> -release = environ.setdefault('DPDK_VERSION', "None")
> -version = release
> +version = environ.setdefault('DPDK_VERSION', "None")
>
>  master_doc = 'index'
>
> diff --git a/doc/guides/meson.build b/doc/guides/meson.build
> index 51f81da2e3..8933d75f6b 100644
> --- a/doc/guides/meson.build
> +++ b/doc/guides/meson.build
> @@ -1,6 +1,7 @@
>  # SPDX-License-Identifier: BSD-3-Clause
>  # Copyright(c) 2018 Intel Corporation
>
> +doc_guides_source_dir = meson.current_source_dir()
>  sphinx = find_program('sphinx-build', required: get_option('enable_docs'))
>
>  if not sphinx.found()
> diff --git a/doc/guides/tools/dts.rst b/doc/guides/tools/dts.rst
> index 846696e14e..21d3d89fc2 100644
> --- a/doc/guides/tools/dts.rst
> +++ b/doc/guides/tools/dts.rst
> @@ -278,7 +278,12 @@ and try not to divert much from it.
>  The :ref:`DTS developer tools <dts_dev_tools>` will issue warnings
>  when some of the basics are not met.
>
> -The code must be properly documented with docstrings.
> +The API documentation, which is a helpful reference when developing, may be accessed
> +in the code directly or generated with the :ref:`API docs build steps <building_api_docs>`.
> +When adding new files or modifying the directory structure, the corresponding changes must
> +be made to DTS api doc sources in ``dts/doc``.
> +
> +Speaking of which, the code must be properly documented with docstrings.
>  The style must conform to the `Google style
>  <https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings>`_.
>  See an example of the style `here
> @@ -413,6 +418,33 @@ the DTS code check and format script.
>  Refer to the script for usage: ``devtools/dts-check-format.sh -h``.
>
>
> +.. _building_api_docs:
> +
> +Building DTS API docs
> +---------------------
> +
> +To build DTS API docs, install the dependencies with Poetry, then enter its shell:
> +
> +.. code-block:: console
> +
> +   poetry install --with docs
> +   poetry shell
> +

The only thing to note here is with newer versions of poetry this will
start to throw warnings because of the way we use poetry and don't
have a root package. It is just a warning message so it shouldn't
cause any real problems, but I believe the way we should be handling
it is passing --no-root into poetry install so that it knows not to
use the root package.

>
> +The documentation is built using the standard DPDK build system. After executing the meson command
> +and entering Poetry's shell, build the documentation with:
> +
> +.. code-block:: console
> +
> +   ninja -C build dts-doc
> +
> +The output is generated in ``build/doc/api/dts/html``.
> +
> +.. Note::
> +
> +   Make sure to fix any Sphinx warnings when adding or updating docstrings. Also make sure to run
> +   the ``devtools/dts-check-format.sh`` script and address any issues it finds.
> +
> +
>  Configuration Schema
>  --------------------
>
> diff --git a/dts/doc/meson.build b/dts/doc/meson.build
> new file mode 100644
> index 0000000000..01b7b51034
> --- /dev/null
> +++ b/dts/doc/meson.build
> @@ -0,0 +1,27 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2023 PANTHEON.tech s.r.o.
> +
> +sphinx = find_program('sphinx-build', required: false)
> +sphinx_apidoc = find_program('sphinx-apidoc', required: false)
> +
> +if not sphinx.found() or not sphinx_apidoc.found()
> +    subdir_done()
> +endif
> +
> +dts_doc_api_build_dir = join_paths(doc_api_build_dir, 'dts')
> +
> +extra_sphinx_args = ['-E', '-c', doc_guides_source_dir, '--dts-root', dts_dir]
> +if get_option('werror')
> +    extra_sphinx_args += '-W'
> +endif
> +
> +htmldir = join_paths(get_option('datadir'), 'doc', 'dpdk', 'dts')
> +dts_api_html = custom_target('dts_api_html',
> +        output: 'html',
> +        command: [sphinx_wrapper, sphinx, meson.project_version(),
> +            meson.current_source_dir(), dts_doc_api_build_dir, extra_sphinx_args],
> +        build_by_default: false,
> +        install: get_option('enable_docs'),
> +        install_dir: htmldir)
> +doc_targets += dts_api_html
> +doc_target_names += 'DTS_API_HTML'
> diff --git a/dts/meson.build b/dts/meson.build
> new file mode 100644
> index 0000000000..e8ce0f06ac
> --- /dev/null
> +++ b/dts/meson.build
> @@ -0,0 +1,16 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2023 PANTHEON.tech s.r.o.
> +
> +doc_targets = []
> +doc_target_names = []
> +dts_dir = meson.current_source_dir()
> +
> +subdir('doc')
> +
> +if doc_targets.length() == 0
> +    message = 'No docs targets found'
> +else
> +    message = 'Built docs:'
> +endif
> +run_target('dts-doc', command: [echo, message, doc_target_names],
> +    depends: doc_targets)
> diff --git a/meson.build b/meson.build
> index 5e161f43e5..001fdcbbbf 100644
> --- a/meson.build
> +++ b/meson.build
> @@ -87,6 +87,7 @@ subdir('app')
>
>  # build docs
>  subdir('doc')
> +subdir('dts')
>
>  # build any examples explicitly requested - useful for developers - and
>  # install any example code into the appropriate install path
> --
> 2.34.1
>
Reviewed-by: Jeremy Spewock <jspewock@iol.unh.edu>