DPDK patches and discussions
 help / color / mirror / Atom feed
From: Dmitry Kozlyuk <dmitry.kozliuk@gmail.com>
To: dev@dpdk.org
Cc: Neil Horman <nhorman@tuxdriver.com>,
	Bruce Richardson <bruce.richardson@intel.com>,
	Thomas Monjalon <thomas@monjalon.net>,
	robin.jarry@6wind.com, Dmitry Kozlyuk <dmitry.kozliuk@gmail.com>
Subject: [dpdk-dev] [RFC PATCH 1/2] pmdinfogen: prototype in Python
Date: Mon, 22 Jun 2020 03:45:02 +0300
Message-ID: <20200622004503.29036-2-dmitry.kozliuk@gmail.com> (raw)
In-Reply-To: <20200622004503.29036-1-dmitry.kozliuk@gmail.com>

* No user-friendly error handling and no bounds checking yet.
* No support for >65K sections case (is it needed?).
* The order of definitions is reversed.

Signed-off-by: Dmitry Kozlyuk <dmitry.kozliuk@gmail.com>
---
 buildtools/elf.py        | 194 +++++++++++++++++++++++++++++++++++++++
 buildtools/pmdinfogen.py | 144 +++++++++++++++++++++++++++++
 2 files changed, 338 insertions(+)
 create mode 100644 buildtools/elf.py
 create mode 100755 buildtools/pmdinfogen.py

diff --git a/buildtools/elf.py b/buildtools/elf.py
new file mode 100644
index 000000000..4085d547b
--- /dev/null
+++ b/buildtools/elf.py
@@ -0,0 +1,194 @@
+import ctypes
+
+
+class ElfIdent(ctypes.Structure):
+    _pack_ = True
+    _fields_ = [
+        ("magic", ctypes.c_char * 4),
+        ("class_", ctypes.c_uint8),
+        ("data", ctypes.c_uint8),
+        ("version", ctypes.c_uint8),
+        ("os_abi", ctypes.c_uint8),
+        ("abi_version", ctypes.c_uint8),
+        ("pad", ctypes.c_uint8 * 7),
+    ]
+
+    @property
+    def is_magic_ok(self):
+        return self.magic.value == b"\x7fELF"
+
+    @property
+    def is_32bit(self):
+        return self.class_ == 1
+
+    @property
+    def is_big_endian(self):
+        return self.data == 2
+
+    def define_structures(self):
+        base_type = ctypes.LittleEndianStructure
+        if self.is_big_endian:
+            base = ctypes.BigEndianStructure
+
+        size_type = ctypes.c_uint64
+        if self.is_32bit:
+            size_type = ctypes.c_uint32
+
+        class FileHeader(base_type):
+            _pack_ = True
+            _fields_ = [
+                ("e_ident", ElfIdent),
+                ("e_type", ctypes.c_uint16),
+                ("e_machine", ctypes.c_uint16),
+                ("e_version", ctypes.c_uint32),
+                ("e_entry", size_type),
+                ("e_phoff", size_type),
+                ("e_shoff", size_type),
+                ("e_flags", ctypes.c_uint32),
+                ("e_ehsize", ctypes.c_uint16),
+                ("e_phentsize", ctypes.c_uint16),
+                ("e_phnum", ctypes.c_uint16),
+                ("e_shentsize", ctypes.c_uint16),
+                ("e_shnum", ctypes.c_uint16),
+                ("e_shstrndx", ctypes.c_uint16),
+            ]
+
+        class SectionHeader(base_type):
+            _pack_ = True
+            _fields_ = [
+                ("sh_name", ctypes.c_uint32),
+                ("sh_type", ctypes.c_uint32),
+                ("sh_flags", size_type),
+                ("sh_addr", size_type),
+                ("sh_offset", size_type),
+                ("sh_size", size_type),
+                ("sh_link", ctypes.c_uint32),
+                ("sh_info", ctypes.c_uint32),
+                ("sh_addralign", size_type),
+                ("sh_entsize", size_type),
+            ]
+
+        class Symbol32(base_type):
+            _pack_ = True
+            _fields_ = [
+                ("st_name", ctypes.c_uint32),
+                ("st_value", ctypes.c_uint32),
+                ("st_size", ctypes.c_uint32),
+                ("st_info", ctypes.c_uint8),
+                ("st_other", ctypes.c_uint8),
+                ("st_shndx", ctypes.c_uint16),
+            ]
+
+        class Symbol64(base_type):
+            _pack_ = True
+            _fields_ = [
+                ("st_name", ctypes.c_uint32),
+                ("st_info", ctypes.c_uint8),
+                ("st_other", ctypes.c_uint8),
+                ("st_shndx", ctypes.c_uint16),
+                ("st_value", ctypes.c_uint64),
+                ("st_size", ctypes.c_uint64),
+            ]
+
+        Symbol = Symbol32 if self.is_32bit else Symbol64
+
+        return FileHeader, SectionHeader, Symbol
+
+
+class Symbol:
+    def __init__(self, image, elf):
+        self._image = image
+        self._elf = elf
+
+
+    @property
+    def address(self):
+        base = self._image._sections[self._elf.st_shndx].sh_offset
+        offset = base + self._elf.st_value
+        memory = ctypes.c_char.from_buffer(self._image._data, offset)
+        return ctypes.addressof(memory)
+
+
+class Image:
+    def __init__(self, data):
+        SHN_UNDEF = 0x0000
+        SHN_XINDEX = 0xFFFF
+
+        ident = ElfIdent.from_buffer(data)
+        ElfFileHeader, ElfSectionHeader, ElfSymbol = ident.define_structures()
+
+        header = ElfFileHeader.from_buffer(data)
+
+        if header.e_shnum == SHN_UNDEF:
+            section = ElfSectionHeader.from_buffer(data, header.e_shoff)
+            sections_num = section.sh_size
+        else:
+            sections_num = header.e_shnum
+        sections_desc = ElfSectionHeader * sections_num
+        sections = sections_desc.from_buffer(data, header.e_shoff)
+
+        if header.e_shstrndx == SHN_XINDEX:
+            strings_index = sections[0].sh_link
+        else:
+            strings_index = header.e_shstrndx
+
+        symtab, strtab = Image._find_symbol_table(data, sections, ElfSymbol)
+
+        self._data = data
+        self._header = header
+        self._sections = sections
+        self._strings = sections[strings_index]
+        self._symtab = symtab
+        self._strtab = strtab
+
+
+    @staticmethod
+    def _find_symbol_table(data, sections, symbol_type):
+        SHT_SYMTAB = 2
+        SHT_SYMTAB_SHNDX = 18
+
+        for section in sections:
+            if section.sh_type == SHT_SYMTAB:
+                symbol_count = section.sh_size // ctypes.sizeof(symbol_type)
+                symtab_desc = symbol_type * symbol_count
+                symtab_offset = section.sh_offset
+                symtab = symtab_desc.from_buffer(data, symtab_offset)
+
+                strtab = sections[section.sh_link].sh_offset
+
+                return symtab, strtab
+
+            # TODO: SHT_SYMTAB_SHNDX?
+
+        raise Exception('no symbol table')
+
+
+    @property
+    def is_big_endian(self):
+        return self._header.e_ident.is_big_endian
+
+
+    def find_symbol(self, name: str, start: Symbol = None):
+        name_size = len(name)
+        name_desc = ctypes.c_char * name_size
+
+        i = 0
+        if start:
+            table_address = ctypes.addressof(self._symtab)
+            start_address = ctypes.addressof(start._elf)
+            i = (start_address - table_address) // ctypes.sizeof(start._elf) + 1
+
+        while i < len(self._symtab):
+            symbol = self._symtab[i]
+
+            name_offset = self._strtab + symbol.st_name
+            if name_offset + name_size > len(self._data):
+                break
+
+            symbol_name = name_desc.from_buffer(self._data, name_offset).value
+            if symbol_name == name.encode():
+                return Symbol(self, symbol)
+
+            i += 1
+
+        return None
diff --git a/buildtools/pmdinfogen.py b/buildtools/pmdinfogen.py
new file mode 100755
index 000000000..74d2e5285
--- /dev/null
+++ b/buildtools/pmdinfogen.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+
+import argparse
+import ctypes
+import elf
+import json
+import mmap
+import os
+import sys
+import tempfile
+
+
+def define_rte_pci_id(is_big_endian):
+    base_type = ctypes.LittleEndianStructure
+    if is_big_endian:
+        base_type = ctypes.BigEndianStructure
+
+    class rte_pci_id(base_type):
+        _pack_ = True
+        _fields_ = [
+            ("class_id", ctypes.c_uint32),
+            ("vendor_id", ctypes.c_uint16),
+            ("device_id", ctypes.c_uint16),
+            ("subsystem_vendor_id", ctypes.c_uint16),
+            ("subsystem_device_id", ctypes.c_uint16),
+        ]
+
+    return rte_pci_id
+
+
+class Driver:
+    OPTIONS = [
+        ("params", "_param_string_export"),
+        ("kmod", "_kmod_dep_export"),
+    ]
+
+    def __init__(self, name, options):
+        self.name = name
+        for key, value in options.items():
+            setattr(self, key, value)
+        self.pci_ids = []
+
+    @classmethod
+    def load(cls, image, symbol):
+        name = ctypes.string_at(symbol.address).decode()
+
+        options = {}
+        for key, suffix in cls.OPTIONS:
+            option_symbol = image.find_symbol(f"__{name}{suffix}")
+            if option_symbol:
+                value = ctypes.string_at(option_symbol.address).decode()
+                options[key] = value
+
+        driver = cls(name, options)
+
+        pci_table_name_symbol = image.find_symbol(f"__{name}_pci_tbl_export")
+        if not pci_table_name_symbol:
+            return driver
+
+        pci_table_name = ctypes.string_at(pci_table_name_symbol.address).decode()
+
+        pci_table_symbol = image.find_symbol(pci_table_name)
+        if not pci_table_symbol:
+            raise Exception('PCI table declared but not defined')
+
+        rte_pci_id = define_rte_pci_id(image.is_big_endian)
+
+        pci_id = rte_pci_id.from_address(pci_table_symbol.address)
+        while pci_id.device_id:
+            driver.pci_ids.append(
+                [
+                    pci_id.vendor_id,
+                    pci_id.device_id,
+                    pci_id.subsystem_vendor_id,
+                    pci_id.subsystem_device_id,
+                ]
+            )
+            pci_id = rte_pci_id.from_address(
+                ctypes.addressof(pci_id) + ctypes.sizeof(pci_id)
+            )
+
+        return driver
+
+    def dump(self, file):
+        dumped = json.dumps(self.__dict__)
+        escaped = dumped.replace('"', '\\"')
+        print(
+            f"const char {self.name}_pmd_info[] __attribute__((used)) = "
+            f'"PMD_INFO_STRING= {escaped}";',
+            file=file,
+        )
+
+
+def get_symbols_by_prefix(image, name):
+    symbol = image.find_symbol(name)
+    while symbol:
+        yield symbol
+        symbol = image.find_symbol(name, symbol)
+
+
+def load_drivers(image):
+    drivers = []
+    for symbol in get_symbols_by_prefix(image, "this_pmd_name"):
+        drivers.append(Driver.load(image, symbol))
+    return drivers
+
+
+def dump_drivers(drivers, file):
+    for driver in drivers:
+        driver.dump(file)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("input", help="input object file path")
+    parser.add_argument("output", help="output C file path")
+    return parser.parse_args()
+
+
+def map_input(path):
+    if path == '-':
+        fd, name = tempfile.mkstemp()
+    else:
+        fd = os.open(path, os.O_RDONLY)
+    return mmap.mmap(fd, 0, access=mmap.ACCESS_COPY)
+
+
+def open_output(path):
+    if path == '-':
+        return sys.stdout
+    return open(path, 'w')
+
+
+def main():
+    args = parse_args()
+    memory = map_input(args.input)
+    image = elf.Image(memory)
+    drivers = load_drivers(image)
+    output = open_output(args.output)
+    dump_drivers(drivers, output)
+
+
+if __name__ == "__main__":
+    main()
-- 
2.25.4


  reply	other threads:[~2020-06-22  0:45 UTC|newest]

Thread overview: 88+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-06-22  0:45 [dpdk-dev] [RFC PATCH 0/2] pmdinfogen: rewrite " Dmitry Kozlyuk
2020-06-22  0:45 ` Dmitry Kozlyuk [this message]
2020-06-22  0:45 ` [dpdk-dev] [RFC PATCH 2/2] build: use Python pmdinfogen Dmitry Kozlyuk
2020-06-22 12:41 ` [dpdk-dev] [RFC PATCH 0/2] pmdinfogen: rewrite in Python Neil Horman
2020-06-22 19:39   ` Dmitry Kozlyuk
2020-06-23 11:28     ` Neil Horman
2020-06-23 11:59       ` Bruce Richardson
2020-07-02  0:07       ` Dmitry Kozlyuk
2020-07-02  0:02 ` [dpdk-dev] [RFC PATCH v2 " Dmitry Kozlyuk
2020-07-02  0:02   ` [dpdk-dev] [RFC PATCH v2 1/3] pmdinfogen: prototype " Dmitry Kozlyuk
2020-07-02  0:02   ` [dpdk-dev] [RFC PATCH v2 2/3] build: use Python pmdinfogen Dmitry Kozlyuk
2020-07-02  0:02   ` [dpdk-dev] [RFC PATCH v2 3/3] doc/linux_gsg: require pyelftools for pmdinfogen Dmitry Kozlyuk
2020-07-06 12:52     ` Neil Horman
2020-07-06 13:24       ` Dmitry Kozlyuk
2020-07-06 16:46         ` Neil Horman
2020-07-08  0:53   ` [dpdk-dev] [PATCH v3 0/4] pmdinfogen: rewrite in Python Dmitry Kozlyuk
2020-07-08  0:53     ` [dpdk-dev] [PATCH v3 1/4] pmdinfogen: add Python implementation Dmitry Kozlyuk
2020-07-08  0:53     ` [dpdk-dev] [PATCH v3 2/4] build: use Python pmdinfogen Dmitry Kozlyuk
2020-07-08  0:53     ` [dpdk-dev] [PATCH v3 3/4] doc/linux_gsg: require pyelftools for pmdinfogen Dmitry Kozlyuk
2020-07-08  0:53     ` [dpdk-dev] [PATCH v3 4/4] pmdinfogen: remove C implementation Dmitry Kozlyuk
2020-07-08 21:23     ` [dpdk-dev] [PATCH v4 0/4] pmdinfogen: rewrite in Python Dmitry Kozlyuk
2020-07-08 21:23       ` [dpdk-dev] [PATCH v4 1/4] pmdinfogen: add Python implementation Dmitry Kozlyuk
2020-07-08 21:23       ` [dpdk-dev] [PATCH v4 2/4] build: use Python pmdinfogen Dmitry Kozlyuk
2020-07-21 14:04         ` Bruce Richardson
2020-07-21 14:59           ` Dmitry Kozlyuk
2020-07-08 21:23       ` [dpdk-dev] [PATCH v4 3/4] doc/linux_gsg: require pyelftools for pmdinfogen Dmitry Kozlyuk
2020-07-21 13:39         ` Bruce Richardson
2020-07-21 14:05           ` Bruce Richardson
2020-07-21 14:04         ` Bruce Richardson
2020-07-08 21:23       ` [dpdk-dev] [PATCH v4 4/4] pmdinfogen: remove C implementation Dmitry Kozlyuk
2020-07-09 10:42       ` [dpdk-dev] [PATCH v4 0/4] pmdinfogen: rewrite in Python Neil Horman
2020-07-21 13:51       ` Bruce Richardson
2020-09-27 21:47       ` [dpdk-dev] [PATCH v5 0/3] " Dmitry Kozlyuk
2020-09-27 21:47         ` [dpdk-dev] [PATCH v5 1/3] pmdinfogen: add Python implementation Dmitry Kozlyuk
2020-09-27 22:05           ` Stephen Hemminger
2020-09-27 21:47         ` [dpdk-dev] [PATCH v5 2/3] build: use Python pmdinfogen Dmitry Kozlyuk
2020-09-27 21:47         ` [dpdk-dev] [PATCH v5 3/3] pmdinfogen: remove C implementation Dmitry Kozlyuk
2020-09-27 23:15           ` Thomas Monjalon
2020-09-28  9:35         ` [dpdk-dev] [PATCH v5 0/3] pmdinfogen: rewrite in Python David Marchand
2020-10-04  1:59         ` [dpdk-dev] [PATCH v6 " Dmitry Kozlyuk
2020-10-04  1:59           ` [dpdk-dev] [PATCH v6 1/3] pmdinfogen: add Python implementation Dmitry Kozlyuk
2020-10-04  1:59           ` [dpdk-dev] [PATCH v6 2/3] build: use Python pmdinfogen Dmitry Kozlyuk
2020-10-04  1:59           ` [dpdk-dev] [PATCH v6 3/3] pmdinfogen: remove C implementation Dmitry Kozlyuk
2020-10-14 14:37           ` [dpdk-dev] [PATCH v6 0/3] pmdinfogen: rewrite in Python Maxime Coquelin
2020-10-14 15:40             ` Dmitry Kozlyuk
2020-10-14 18:31           ` [dpdk-dev] [PATCH v7 " Dmitry Kozlyuk
2020-10-14 18:31             ` [dpdk-dev] [PATCH v7 1/3] pmdinfogen: add Python implementation Dmitry Kozlyuk
2020-10-14 18:31             ` [dpdk-dev] [PATCH v7 2/3] build: use Python pmdinfogen Dmitry Kozlyuk
2020-10-14 18:31             ` [dpdk-dev] [PATCH v7 3/3] pmdinfogen: remove C implementation Dmitry Kozlyuk
2020-10-20 16:02             ` [dpdk-dev] [PATCH v7 0/3] pmdinfogen: rewrite in Python David Marchand
2020-10-20 17:45               ` Dmitry Kozlyuk
2020-10-20 22:09               ` Dmitry Kozlyuk
2020-10-20 17:44             ` [dpdk-dev] [PATCH v8 " Dmitry Kozlyuk
2020-10-20 17:44               ` [dpdk-dev] [PATCH v8 1/3] pmdinfogen: add Python implementation Dmitry Kozlyuk
2020-10-20 17:44               ` [dpdk-dev] [PATCH v8 2/3] build: use Python pmdinfogen Dmitry Kozlyuk
2020-10-21  9:00                 ` Bruce Richardson
2021-01-20  0:05                 ` Thomas Monjalon
2021-01-20  7:23                   ` Dmitry Kozlyuk
2021-01-20 10:24                     ` Thomas Monjalon
2021-01-22 20:31                       ` Dmitry Kozlyuk
2021-01-22 20:57                         ` Thomas Monjalon
2021-01-22 22:24                           ` Dmitry Kozlyuk
2021-01-23 11:38                             ` Thomas Monjalon
2021-01-24 20:52                               ` Dmitry Kozlyuk
2021-01-25  9:25                               ` Kinsella, Ray
2021-01-25 10:01                                 ` Kinsella, Ray
2021-01-25 10:29                                   ` David Marchand
2021-01-25 10:46                                     ` Kinsella, Ray
2021-01-25 11:03                                       ` Thomas Monjalon
2021-01-25 10:05                                 ` Dmitry Kozlyuk
2021-01-25 10:11                                   ` Kinsella, Ray
2021-01-25 10:31                                     ` Dmitry Kozlyuk
2020-10-20 17:44               ` [dpdk-dev] [PATCH v8 3/3] pmdinfogen: remove C implementation Dmitry Kozlyuk
2020-10-26 16:46                 ` Jie Zhou
2021-01-22 22:43               ` [dpdk-dev] [PATCH v9 0/3] pmdinfogen: rewrite in Python Dmitry Kozlyuk
2021-01-22 22:43                 ` [dpdk-dev] [PATCH v9 1/3] pmdinfogen: add Python implementation Dmitry Kozlyuk
2021-01-22 22:43                 ` [dpdk-dev] [PATCH v9 2/3] build: use Python pmdinfogen Dmitry Kozlyuk
2021-01-22 22:43                 ` [dpdk-dev] [PATCH v9 3/3] pmdinfogen: remove C implementation Dmitry Kozlyuk
2021-01-24 20:51                 ` [dpdk-dev] [PATCH v10 0/3] pmdinfogen: rewrite in Python Dmitry Kozlyuk
2021-01-24 20:51                   ` [dpdk-dev] [PATCH v10 1/3] pmdinfogen: add Python implementation Dmitry Kozlyuk
2021-01-24 20:51                   ` [dpdk-dev] [PATCH v10 2/3] build: use Python pmdinfogen Dmitry Kozlyuk
2021-01-25 10:12                     ` Thomas Monjalon
2021-01-24 20:51                   ` [dpdk-dev] [PATCH v10 3/3] pmdinfogen: remove C implementation Dmitry Kozlyuk
2021-01-25 13:13                   ` [dpdk-dev] [PATCH v10 0/3] pmdinfogen: rewrite in Python Thomas Monjalon
2021-01-25 16:08                     ` Brandon Lo
2021-02-02  8:48                       ` Tal Shnaiderman
2021-01-25 18:51                   ` Ali Alnubani
2021-01-25 22:15                     ` Dmitry Kozlyuk

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20200622004503.29036-2-dmitry.kozliuk@gmail.com \
    --to=dmitry.kozliuk@gmail.com \
    --cc=bruce.richardson@intel.com \
    --cc=dev@dpdk.org \
    --cc=nhorman@tuxdriver.com \
    --cc=robin.jarry@6wind.com \
    --cc=thomas@monjalon.net \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

DPDK patches and discussions

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://inbox.dpdk.org/dev/0 dev/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 dev dev/ https://inbox.dpdk.org/dev \
		dev@dpdk.org
	public-inbox-index dev

Example config snippet for mirrors.
Newsgroup available over NNTP:
	nntp://inbox.dpdk.org/inbox.dpdk.dev


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git