waf: Add configurations with different modules.

Update #3351

CONTRIBUTING.md

@@ -67,8 +67,7 @@ freebsd-to-rtems.py [args]
 ```
 
 In its default mode of operation, freebsd-to-rtems.py is used to copy code
-from FreeBSD to the rtems-libbsd tree and perform transformations.  In forward
+from FreeBSD to the rtems-libbsd tree and perform transformations.
-mode, the script may be requested to just generate the Waf script.
 
 In *reverse mode*, this script undoes those transformations and copies
 the source code back to the *master* FreeBSD tree. This allows us to do
@@ -126,9 +125,9 @@ How to import code from FreeBSD
 * Run `./freebsd-to-rtems.py -R`
 * Run `./freebsd-to-rtems.py`
 * Run `git status` and make sure your working directory is clean.  If you see modified files, then the `freebsd-to-rtems.py` script needs to be fixed first.
-* Add the files to import to `libbsd.py`.
+* Add the files to import to `libbsd.py` and your intended build set (for example `buildset/default.ini`.
 * Run `./freebsd-to-rtems.py`
-* Immediately check in the imported files without the changes to `libbsd_waf.py`.  Do not touch the imported files yourself at this point.
+* Immediately check in the imported files without the changes to `libbsd.py` and the buildsets.  Do not touch the imported files yourself at this point.
 * Port the imported files to RTEMS.  See 'Rules for Modifying FreeBSD Source'.
 * Add a test to the testsuite if possible.
 * Run `./create-kernel-namespace.sh` if you imported kernel space headers.  Add only your new defines via `git add -p rtemsbsd/include/machine/rtems-bsd-kernel-namespace.h`.

README.waf

@@ -73,7 +73,8 @@ cd rtems-libbsd
 git submodule init
 git submodule update rtems_waf
 waf configure --prefix="$sandbox/rtems-4.12" \
-  --rtems-bsps=arm/xilinx_zynq_a9_qemu
+  --rtems-bsps=arm/xilinx_zynq_a9_qemu \
+  --buildset=buildset/default.ini
 waf
 waf install
 qemu-system-arm -no-reboot -serial null -serial mon:stdio -net none \
@@ -132,12 +133,17 @@ qemu-system-arm -no-reboot -serial null -serial mon:stdio -net none \
    '--rtems-archs=arm,sparc,i386' or
    '--rtems-bsps=arm/xilinx_zynq_a9_qemu,sparc/sis,i386/pc586' to build for
    more than BSP at a time.  Note, you must provide the architecture and BSP as
-   a pair. Providing just the BSP name will fail:
+   a pair. Providing just the BSP name will fail. This call also explicitly
+   provides a buildset via the '--buildset=buildset/default.ini' option. If no
+   buildset is provided the default one (which is the same as the one provided
+   explicitly here) will be used. You can also provide multiple buildsets as a
+   coma separated list or via multiple '--buildset=x' options.
 
    $ cd "$sandbox"
    $ cd rtems-libbsd
    $ waf configure --prefix="$sandbox/rtems-4.12" \
-       --rtems-bsps=arm/xilinx_zynq_a9_qemu
+       --rtems-bsps=arm/xilinx_zynq_a9_qemu \
+       --buildset=buildset/default.ini
 
 8. Build and install.  The LibBSD package will be installed into the prefix
    provided to configure:

builder.py

@@ -44,6 +44,7 @@ import getopt
 import filecmp
 import difflib
 import codecs
+import copy
 
 #
 # Global controls.
@@ -614,15 +615,11 @@ class Module(object):
     def __init__(self, manager, name, enabled = True):
         self.manager = manager
         self.name = name
-        self.enabled = enabled
         self.conditionalOn = "none"
         self.files = []
         self.cpuDependentSourceFiles = {}
         self.dependencies = []
 
-    def isEnabled(self):
-        return self.enabled
-
     def initCPUDependencies(self, cpu):
         if cpu not in self.cpuDependentSourceFiles:
             self.cpuDependentSourceFiles[cpu] = []
@@ -791,15 +788,19 @@ class ModuleManager(object):
             self.modules[m].processSource(direction)
 
     def setConfiguration(self, config):
-        self.configuration = config
+        self.configuration = copy.deepcopy(config)
 
     def getConfiguration(self):
         return self.configuration
 
+    def updateConfiguration(self, config):
+        self.configuration.update(config)
+
     def setModuleConfigiuration(self):
         mods = sorted(self.modules.keys())
         self.configuration['modules'] = mods
-        self.configuration['modules-enabled'] = [m for m in mods if self.modules[m].isEnabled()]
+        # Enabled modules are overwritten by config file. Default to all.
+        self.configuration['modules-enabled'] = mods
 
     def generateBuild(self, only_enabled=True):
         modules_to_process = self.getEnabledModules()

buildset/default.ini

@@ -0,0 +1,58 @@
+#
+# Default configuration.
+#
+
+[general]
+name = default
+
+[modules]
+altq = on
+base = on
+cam = on
+contrib_expat = on
+contrib_libpcap = on
+crypto = on
+crypto_openssl = on
+dev_input = on
+dev_net = on
+dev_nic = on
+dev_nic_broadcomm = on
+dev_nic_dc = on
+dev_nic_e1000 = on
+dev_nic_fxp = on
+dev_nic_re = on
+dev_nic_smc = on
+dev_usb = on
+dev_usb_controller = on
+dev_usb_controller_bbb = on
+dev_usb_input = on
+dev_usb_net = on
+dev_usb_quirk = on
+dev_usb_serial = on
+dev_usb_storage = on
+dev_usb_wlan = on
+dev_wlan_rtwn = on
+dhcpcd = on
+dpaa = on
+evdev = on
+fdt = on
+in_cksum = on
+ipfw = on
+mdnsresponder = on
+mghttpd = on
+mmc = on
+mmc_ti = on
+net = on
+net80211 = on
+netinet = on
+netinet6 = on
+opencrypto = on
+pci = on
+pf = on
+rtems = on
+tests = on
+tty = on
+user_space = on
+user_space_wlanstats = on
+usr_sbin_tcpdump = on
+usr_sbin_wpa_supplicant = on

buildset/sample.ini

@@ -0,0 +1,10 @@
+#
+# Currently this is mostly a sample configuration.
+#
+
+[general]
+name = sample
+extends = default.ini
+
+[modules]
+dev_nic_broadcomm = off

libbsd.py

@@ -1996,7 +1996,7 @@ class netinet6(builder.Module):
 class netipsec(builder.Module):
 
     def __init__(self, manager):
-        super(netipsec, self).__init__(manager, type(self).__name__, enabled = False)
+        super(netipsec, self).__init__(manager, type(self).__name__)
 
     def generate(self):
         mm = self.manager

rtems_waf

@@ -1 +1 @@
-Subproject commit d793097d2032ca0ab2d5cf93fa84730865db6c03
+Subproject commit f490cd3197d30476fb919ca0702c5a51fe2960bc

waf_libbsd.py

@@ -141,12 +141,6 @@ class Builder(builder.ModuleManager):
             import pprint
             pprint.pprint(self.data)
 
-    def init(self, ctx):
-        pass
-
-    def options(self, opt):
-        pass
-
     def bsp_configure(self, conf, arch_bsp):
         if 'configure' in self.data:
             for cfg in self.data['configure']:
@@ -156,9 +150,6 @@ class Builder(builder.ModuleManager):
                                includes = conf.env.IFLAGS,
                                mandatory = False)
 
-    def configure(self, conf):
-        pass
-
     def build(self, bld):
         #
         # Localize the config.

wscript

@@ -43,23 +43,126 @@ except:
 
 import libbsd
 import waf_libbsd
+import os.path
+import runpy
+import sys
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+import waflib.Options
 
-builder = None
+builders = {}
 
-def create_builder():
+BUILDSET_DIR = "buildset"
-    global builder
+BUILDSET_DEFAULT = "buildset/default.ini"
-    if builder is None:
+
+def load_ini(conf, f):
+    ini = configparser.ConfigParser()
+    ini.read(f)
+    if not ini.has_section('general'):
+        conf.fatal("'{}' is missing a general section.".format(f))
+    if not ini.has_option('general', 'name'):
+        conf.fatal("'{}' is missing a general/name.".format(f))
+    if ini.has_option('general', 'extends'):
+        extends = ini.get('general', 'extends')
+        extendfile = None
+        basepath = os.path.dirname(f)
+        if os.path.isfile(os.path.join(basepath, extends)):
+            extendfile = os.path.join(basepath, extends)
+        elif os.path.isfile(os.path.join(BUILDSET_DIR, extends)):
+            extendfile = os.path.join(BUILDSET_DIR, extends)
+        else:
+            conf.fatal("'{}': Invalid file given for general/extends:'{}'"
+                .format(f, extends))
+        base = load_ini(conf, extendfile)
+        for s in ini.sections():
+            if not base.has_section(s):
+                base.add_section(s)
+            for o in ini.options(s):
+                val = ini.get(s, o)
+                base.set(s, o, val)
+        ini = base
+    return ini
+
+def load_config(conf, f):
+    ini = load_ini(conf, f)
+    config = {}
+
+    config['name'] = ini.get('general', 'name')
+
+    config['modules-enabled'] = []
+    mods = []
+    if ini.has_section('modules'):
+        mods = ini.options('modules')
+    for mod in mods:
+        if ini.getboolean('modules', mod):
+            config['modules-enabled'].append(mod)
+    return config
+
+def update_builders(ctx, buildset_opt):
+    global builders
+    builders = {}
+
+    buildsets = []
+    if buildset_opt == []:
+        buildset_opt.append(BUILDSET_DEFAULT)
+    for bs in buildset_opt:
+        if os.path.isdir(bs):
+            for f in os.listdir(bs):
+                if f[-4:] == ".ini":
+                    buildsets += [os.path.join(bs,f)]
+        else:
+            for f in bs.split(','):
+                buildsets += [f]
+
+    for bs in buildsets:
         builder = waf_libbsd.Builder()
         libbsd.load(builder)
+        bsconfig = load_config(ctx, bs)
+        bsname = bsconfig['name']
+        builder.updateConfiguration(bsconfig)
         builder.generate(rtems_version)
+        builders[bsname]=builder
+
+def bsp_init(ctx, env, contexts):
+    # This function generates the builders and adds build-xxx, clean-xxx and
+    # install-xxx targets for them.
+
+    if not 'buildset' in env.options:
+        # This happens if 'waf configure' hasn't been executed. In that case we
+        # create the builders during the configure phase. After the first time
+        # 'waf configure' is executed 'buildset' is read from the .lock_xxx
+        # file. In that case the builders are overwritten during configure
+        # phase. This is not really the cleanest solution but it works.
+        return
+
+    update_builders(ctx, env.options['buildset'])
+    for builder in builders:
+        # Update the contextes for build variants
+        for y in contexts:
+            newcmd = y.cmd + '-' + builder
+            newvariant = y.variant + '-' + builder
+            class context(y):
+                cmd = newcmd
+                variant = newvariant
+                libbsd_buildset_name = builder
+
+    # Transform the commands to per build variant commands
+    commands = []
+    for cmd in waflib.Options.commands:
+        if cmd.startswith(('build', 'clean', 'install')):
+            for builder in builders:
+                commands += [cmd + '-' + builder]
+        else:
+            commands += [cmd]
+    waflib.Options.commands = commands
 
 def init(ctx):
-    create_builder();
+    rtems.init(ctx, version = rtems_version, long_commands = True,
-    rtems.init(ctx, version = rtems_version, long_commands = True)
+               bsp_init = bsp_init)
-    builder.init(ctx)
 
 def options(opt):
-    create_builder();
     rtems.options(opt)
     opt.add_option("--enable-auto-regen",
                    action = "store_true",
@@ -85,20 +188,29 @@ def options(opt):
                    default = "2",
                    dest = "optimization",
                    help = "Set optimization level to OPTIMIZATION (-On compiler flag). Default is 2 (-O2).")
-    builder.options(opt)
+    opt.add_option("--buildset",
+                   action = "append",
+                   default = [],
+                   dest = "buildset",
+                   help = "Select build sets to build. If set to a directory, all .ini file in this directory will be used.")
 
 def bsp_configure(conf, arch_bsp):
-    create_builder();
     conf.check(header_name = "dlfcn.h", features = "c")
     conf.check(header_name = "rtems/pci.h", features = "c", mandatory = False)
     if not rtems.check_posix(conf):
         conf.fatal("RTEMS kernel POSIX support is disabled; configure RTEMS with --enable-posix")
     if rtems.check_networking(conf):
         conf.fatal("RTEMS kernel contains the old network support; configure RTEMS with --disable-networking")
-    builder.bsp_configure(conf, arch_bsp)
+    env = conf.env.derive()
+    for builder in builders:
+        ab = conf.env.RTEMS_ARCH_BSP
+        variant = ab + "-" + builder
+        conf.msg('Configure variant: ', variant)
+        conf.setenv(variant, env)
+        builders[builder].bsp_configure(conf, arch_bsp)
+        conf.setenv(ab)
 
 def configure(conf):
-    create_builder();
     if conf.options.auto_regen:
         conf.find_program("lex", mandatory = True)
         conf.find_program("rpcgen", mandatory = True)
@@ -108,10 +220,12 @@ def configure(conf):
     conf.env.NET_CONFIG = conf.options.net_config
     conf.env.FREEBSD_OPTIONS =conf.options.freebsd_options
     conf.env.OPTIMIZATION = conf.options.optimization
+    conf.env.BUILDSET = conf.options.buildset
+    if len(conf.env.BUILDSET) == 0:
+        conf.env.BUILDSET += [BUILDSET_DEFAULT]
+    update_builders(conf, conf.env.BUILDSET)
     rtems.configure(conf, bsp_configure)
-    builder.configure(conf)
 
 def build(bld):
-    create_builder();
     rtems.build(bld)
-    builder.build(bld)
+    builders[bld.libbsd_buildset_name].build(bld)