[OpenWrt-Devel] [PATCH] scripts: rewritten dumpinfo.py from buildbot.git

[Paul Spooren]

The dumpinfo script parses `target/linux/` for targets and architectures
and prints them to stdout. This is currently used by the buildbot to
automatically generate the buildjobs.

This script is a Python3 rewrite of the Perl script dumpinfo.pl[0] with
the additional feature of allowing to run only on a specific target.

Running only on a specific target can be desireable if a target is added
via feeds.conf. This way only that target is added as a buildjob.

The motivation to rewrite this script in Python is that the buildbot
uses Python which allowing it to directly import the script instead of
running it as a subprocess.

The motivation to add this script to openwrt.git instead of buildbot.git
is that the script is generic enough to be used for other purposes and
as the buildbot clones openwrt.git anyway, it does not add additional
files.

The `architecture` output is not exactly the same as for the original
script as the latter does not sort the targets before printing, as the
Python implementation does.

CC: Jo-Philipp Wich <jo at mein.io>

[0]: https://git.openwrt.org/?p=buildbot.git;a=blob;f=scripts/dumpinfo.pl;h=aa97f8d60379076a41b968402e9337cea824ece5;hb=HEAD

Signed-off-by: Paul Spooren <mail at aparcar.org>
---
 scripts/dumpinfo.py | 143 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 143 insertions(+)
 create mode 100755 scripts/dumpinfo.py

diff --git a/scripts/dumpinfo.py b/scripts/dumpinfo.py
new file mode 100755
index 0000000000..41b7c95b85
--- /dev/null
+++ b/scripts/dumpinfo.py
@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+
+from os import environ
+from pathlib import Path
+import subprocess
+import sys
+
+topdir = Path().cwd()
+
+targets = {}
+architectures = {}
+
+
+def eprint(msg: str):
+    """Print message to stderr and add newline
+
+    Args:
+        msg (str): Message to print
+    """
+    sys.stderr.write(msg + "\n")
+
+
+def get_make_values(cmd: list, target_dir: Path) -> list:
+    """Get Makefile values
+
+    Args:
+        cmd (list): Values to retrive
+        target_dir (Path): Path to run make
+
+    Returns:
+        (list) Makefile output split by newlines
+    """
+    return subprocess.run(
+        ["make", "-C", target_dir, "--no-print-directory", "TARGET_BUILD=1", "DUMP=1"]
+        + cmd,
+        capture_output=True,
+        cwd=topdir,
+        text=True,
+        env={**environ.copy(), "TOPDIR": topdir, "INCLUDE_DIR": topdir / "include"},
+    ).stdout.splitlines()
+
+
+def parse_targetinfo(target_dir: Path, subtarget: str):
+    """Parse target information like arch and features
+
+    After parsing the variables `architectures` and `targets` are automatically filled
+
+    Args:
+        target_dir (Path): Path to run make
+        subtarget (str): Subtarget to parse
+    """
+    result = get_make_values([f"SUBTARGET={subtarget}"], target_dir)
+    for line in result:
+        if line.startswith("Target:"):
+            target = line.split(": ")[1]
+        elif line.startswith("Target-Arch-Packages:"):
+            arch = line.split(": ")[1]
+        elif line.startswith("Target-Features:"):
+            features = line.split(": ")[1]
+        elif line == "@@":
+            if not (target and arch):
+                eprint(f"WARNING: {target} target or arch missing")
+                continue
+
+            if "broken" in features:
+                eprint(f"INFO: {target} skipp broken")
+                continue
+
+            if "source-only" in features:
+                eprint(f"INFO: {target} skipp source-only")
+                continue
+
+            if arch not in architectures:
+                architectures[arch] = set()
+
+            architectures[arch].add(target)
+            targets[target] = arch
+
+
+def get_targetinfo(target: str = "*"):
+    """Get information of all or specific target
+
+    Finds subtargets of target(s) and run `parse_targetinfo`
+
+    Args:
+        target (str): If set limit parsing to single target
+    """
+    for target_makefile in topdir.rglob(f"target/linux/{target}/Makefile"):
+        target_dir = target_makefile.parent
+        cmd = ["val.BOARD", "val.SUBTARGETS", "val.FEATURES"]
+        result = get_make_values(cmd, target_dir)
+
+        if len(result) != 3:
+            eprint(f"WARNING: {target_makefile} seems broken")
+            continue
+
+        target, subtargets, features = result
+        features = set(features.split())
+
+        if "broken" in features:
+            eprint(f"INFO: {target} skip broken")
+            continue
+
+        if "source-only" in features:
+            eprint(f"INFO: {target} skip source-only")
+            continue
+
+        if not subtargets:
+            eprint(f"INFO: {target} has no subtargets defined so use generic")
+            subtargets = ["generic"]
+        else:
+            subtargets = subtargets.split()
+
+        for subtarget in subtargets:
+            parse_targetinfo(target_dir, subtarget)
+
+
+def usage():
+    """Print usage and quits"""
+    print(f"Usage: {sys.argv[0]} targets [target]")
+    print(f"Usage: {sys.argv[0]} architectures [target]")
+    quit()
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 1:
+        usage()
+    if len(sys.argv) == 3:
+        target = sys.argv[2]
+    else:
+        target = "*"
+
+    if sys.argv[1] == "targets":
+        get_targetinfo(target)
+        for target, arch in sorted(targets.items()):
+            print(f"{target} {arch}")
+    elif sys.argv[1] == "architectures":
+        get_targetinfo(target)
+        print(architectures)
+        for arch, subtargets in sorted(architectures.items()):
+            print(f"{arch} {' '.join(sorted(subtargets))}")
+    else:
+        usage()
-- 
2.24.0.rc1


_______________________________________________
openwrt-devel mailing list
openwrt-devel at lists.openwrt.org
https://lists.openwrt.org/mailman/listinfo/openwrt-devel