waf: Implement module dependency checking in the build system

- Do not build a test if a dependency is not enabled.

- Perform a dependency check and generate an error if an enabled module
  depends on a disabled module.

Closes #4077

builder.py

@@ -42,6 +42,11 @@ import os
 import re
 import sys
 
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+
 #
 # Global controls.
 #
@@ -63,6 +68,9 @@ verboseDetail = 2
 verboseMoreDetail = 3
 verboseDebug = 4
 
+BUILDSET_DIR = "buildset"
+BUILDSET_DEFAULT = "buildset/default.ini"
+
 
 def verbose(level=verboseInfo):
     return verboseLevel >= level
@@ -758,8 +766,10 @@ class File(object):
         state = state and (self.pathComposer == self.pathComposer)
         state = state and (self.originPath == self.originPath)
         state = state and (self.forwardConverter == self.forwardConverter)
-        state = state and (self.self.reverseConverter == self.self.reverseConverter)
-        state = state and (self.buildSystemComposer == self.buildSystemComposer)
+        state = state and (self.self.reverseConverter
+                           == self.self.reverseConverter)
+        state = state and (self.buildSystemComposer
+                           == self.buildSystemComposer)
         return state
 
     def processSource(self, forward):
@@ -791,13 +801,12 @@ class Module(object):
     def __init__(self, manager, name, enabled=True):
         self.manager = manager
         self.name = name
-        self.conditionalOn = "none"
         self.files = []
         self.cpuDependentSourceFiles = {}
         self.dependencies = []
 
     def __str__(self):
-        out = [self.name + ': conditional-on=' + self.conditionalOn]
+        out = [self.name + ':']
         if len(self.dependencies) > 0:
             out += [' Deps: ' + str(len(self.dependencies))]
             out += ['  ' + type(d).__name__ for d in self.dependencies]
@@ -978,13 +987,16 @@ class Module(object):
                               NoConverter(), assertSourceFile,
                               sourceFileBuildComposer)
 
-    def addTest(self, testFragementComposer):
+    def addTest(self, testFragementComposer, dependencies=[]):
         self.files += [
             File('user', testFragementComposer.testName, PathComposer(),
                  NoConverter(), NoConverter(), testFragementComposer)
         ]
+        self.dependencies += dependencies
 
     def addDependency(self, dep):
+        if not isinstance(dep, str):
+            raise TypeError('dependencies are a string: %s' % (self.name))
         self.dependencies += [dep]
 
 
@@ -1007,18 +1019,75 @@ class ModuleManager(object):
             out += [str(self.modules[m]), '']
         return os.linesep.join(out)
 
+    def _loadIni(self, ini_file):
+        if not os.path.exists(ini_file):
+            raise FileNotFoundError('file not found: %s' % (ini_file))
+        ini = configparser.ConfigParser()
+        ini.read(ini_file)
+        if not ini.has_section('general'):
+            raise Exception(
+                "'{}' is missing a general section.".format(ini_file))
+        if not ini.has_option('general', 'name'):
+            raise Exception("'{}' is missing a general/name.".format(ini_file))
+        if ini.has_option('general', 'extends'):
+            extends = ini.get('general', 'extends')
+            extendfile = None
+            basepath = os.path.dirname(ini_file)
+            if os.path.isfile(os.path.join(basepath, extends)):
+                extendfile = os.path.join(basepath, extends)
+            elif os.path.isfile(os.path.join(BUILDSET_DIR, extends)):
+                extendfile = os.path.join(BUILDSET_DIR, extends)
+            else:
+                raise Exception(
+                    "'{}': Invalid file given for general/extends:'{}'".format(
+                        ini_file, extends))
+            base = self._loadIni(extendfile)
+            for s in ini.sections():
+                if not base.has_section(s):
+                    base.add_section(s)
+                for o in ini.options(s):
+                    val = ini.get(s, o)
+                    base.set(s, o, val)
+            ini = base
+        return ini
+
+    def _checkDependencies(self):
+        enabled_modules = self.getEnabledModules()
+        enabled_modules.remove('tests')
+        for mod in enabled_modules:
+            if mod not in self.modules:
+                raise KeyError('enabled module not found: %s' % (mod))
+            for dep in self.modules[mod].dependencies:
+                if dep not in self.modules:
+                    print(type(dep))
+                    raise KeyError('dependent module not found: %s' % (dep))
+                if dep not in enabled_modules:
+                    raise Exception('module "%s" dependency "%s" not enabled' %
+                                    (mod, dep))
+
     def getAllModules(self):
         if 'modules' in self.configuration:
-            return self.configuration['modules']
+            return sorted(self.configuration['modules'])
         return []
 
     def getEnabledModules(self):
         if 'modules-enabled' in self.configuration:
-            return self.configuration['modules-enabled']
+            return sorted(self.configuration['modules-enabled'])
         return []
 
     def addModule(self, module):
-        self.modules[module.name] = module
+        name = module.name
+        if name in self.modules:
+            raise KeyError('module already added: %' % (name))
+        self.modules[name] = module
+        if 'modules' not in self.configuration:
+            self.configuration['modules'] = []
+        if 'modules-enabled' not in self.configuration:
+            self.configuration['modules-enabled'] = []
+        self.configuration['modules'] += [name]
+        self.configuration['modules-enabled'] += [name]
+        self.configuration['modules'].sort()
+        self.configuration['modules-enabled'].sort
 
     def processSource(self, direction):
         if verbose(verboseDetail):
@@ -1032,15 +1101,6 @@ class ModuleManager(object):
     def getConfiguration(self):
         return copy.deepcopy(self.configuration)
 
-    def updateConfiguration(self, config):
-        self.configuration.update(config)
-
-    def setModuleConfigiuration(self):
-        mods = sorted(self.modules.keys())
-        self.configuration['modules'] = mods
-        # Enabled modules are overwritten by config file. Default to all.
-        self.configuration['modules-enabled'] = mods
-
     def generateBuild(self, only_enabled=True):
         modules_to_process = self.getEnabledModules()
         # Used for copy between FreeBSD and RTEMS
@@ -1050,6 +1110,7 @@ class ModuleManager(object):
             if m not in self.modules:
                 raise KeyError('enabled module not registered: %s' % (m))
             self.modules[m].generate()
+        self._checkDependencies()
 
     def duplicateCheck(self):
         dups = []
@@ -1065,6 +1126,25 @@ class ModuleManager(object):
                             dups += [(m, mod, fm.getPath(), fm.getSpace())]
         return dups
 
+    def loadConfig(self, config=BUILDSET_DEFAULT):
+        if 'name' in self.configuration:
+            raise KeyError('configuration already loaded: %s (%s)' % \
+                           (self.configuration['name'], config))
+        ini = self._loadIni(config)
+        self.configuration['name'] = ini.get('general', 'name')
+        self.configuration['modules-enabled'] = []
+        mods = []
+        if ini.has_section('modules'):
+            mods = ini.options('modules')
+        for mod in mods:
+            if ini.getboolean('modules', mod):
+                self.configuration['modules-enabled'].append(mod)
+
+    def getName(self):
+        if 'name' not in self.configuration:
+            raise KeyError('configuration not loaded')
+        return self.configuration['name']
+
     def setGenerators(self):
         self.generator['convert'] = Converter
         self.generator['no-convert'] = NoConverter

freebsd-to-rtems.py

@@ -150,6 +150,7 @@ try:
 
     build = builder.ModuleManager()
     libbsd.load(build)
+    build.loadConfig()
     build.generateBuild(only_enabled=False)
 
     dups = build.duplicateCheck()

libbsd.py

@@ -829,7 +829,7 @@ class dev_usb_controller(builder.Module):
 
     def generate(self):
         mm = self.manager
-        self.addDependency(mm['dev_usb'])
+        self.addDependency('dev_usb')
         self.addKernelSpaceHeaderFiles(
             [
                 'sys/dev/usb/controller/ohci.h',
@@ -868,7 +868,7 @@ class dev_usb_input(builder.Module):
 
     def generate(self):
         mm = self.manager
-        self.addDependency(mm['dev_usb'])
+        self.addDependency('dev_usb')
         self.addKernelSpaceHeaderFiles(
             [
                 'sys/dev/usb/input/usb_rdesc.h',
@@ -896,7 +896,7 @@ class dev_usb_net(builder.Module):
 
     def generate(self):
         mm = self.manager
-        self.addDependency(mm['dev_usb'])
+        self.addDependency('dev_usb')
         self.addKernelSpaceHeaderFiles(
             [
                 'sys/dev/usb/net/if_auereg.h',
@@ -946,7 +946,7 @@ class dev_usb_quirk(builder.Module):
 
     def generate(self):
         mm = self.manager
-        self.addDependency(mm['dev_usb'])
+        self.addDependency('dev_usb')
         self.addKernelSpaceHeaderFiles(
             [
                 'sys/dev/usb/quirk/usb_quirk.h',
@@ -969,7 +969,7 @@ class dev_usb_serial(builder.Module):
 
     def generate(self):
         mm = self.manager
-        self.addDependency(mm['dev_usb'])
+        self.addDependency('dev_usb')
         self.addKernelSpaceHeaderFiles(
             [
                 'sys/dev/usb/serial/uftdi_reg.h',
@@ -1014,7 +1014,7 @@ class dev_usb_storage(builder.Module):
 
     def generate(self):
         mm = self.manager
-        self.addDependency(mm['dev_usb'])
+        self.addDependency('dev_usb')
         self.addKernelSpaceSourceFiles(
             [
                 'sys/dev/usb/storage/umass.c',
@@ -1032,7 +1032,7 @@ class dev_usb_controller_bbb(builder.Module):
 
     def generate(self):
         mm = self.manager
-        self.addDependency(mm['dev_usb'])
+        self.addDependency('dev_usb')
         self.addKernelSpaceHeaderFiles(
             [
                 'sys/arm/ti/ti_cpuid.h',
@@ -1069,7 +1069,7 @@ class dev_usb_wlan(builder.Module):
 
     def generate(self):
         mm = self.manager
-        self.addDependency(mm['dev_usb'])
+        self.addDependency('dev_usb')
         self.addKernelSpaceHeaderFiles(
             [
                 'sys/dev/usb/wlan/if_rsureg.h',
@@ -1119,7 +1119,7 @@ class dev_wlan_rtwn(builder.Module):
 
     def generate(self):
         mm = self.manager
-        self.addDependency(mm['dev_usb'])
+        self.addDependency('dev_usb')
         self.addKernelSpaceHeaderFiles(
             [
                 'sys/dev/rtwn/if_rtwn_beacon.h',
@@ -2738,7 +2738,7 @@ class nfsv2(builder.Module):
 
     def generate(self):
         mm = self.manager
-        self.addDependency(mm['rpc_user'])
+        self.addDependency('rpc_user')
         self.addRTEMSUserSourceFiles(
             [
                 'nfsclient/mount_prot_xdr.c',
@@ -3131,7 +3131,7 @@ class crypto_openssl(builder.Module):
 
     def generate(self):
         mm = self.manager
-        self.addDependency(mm['user_space'])
+        self.addDependency('user_space')
         self.addUserSpaceHeaderFiles(
             [
                 'crypto/openssl/crypto/aes/aes_locl.h',
@@ -4108,7 +4108,7 @@ class usr_bin_openssl(builder.Module):
 
     def generate(self):
         mm = self.manager
-        self.addDependency(mm['crypto_openssl'])
+        self.addDependency('crypto_openssl')
         self.addUserSpaceHeaderFiles(
             [
                 'crypto/openssl/apps/apps.h',
@@ -5161,7 +5161,8 @@ class tests(builder.Module):
     def generate(self):
         mm = self.manager
         self.addTest(mm.generator['test']('epoch01', ['test_main'], extraLibs = ['rtemstest']))
-        self.addTest(mm.generator['test']('nfs01', ['test_main'], netTest = True))
+        self.addTest(mm.generator['test']('nfs01', ['test_main'], netTest = True),
+                     ['nfsv2'])
         self.addTest(mm.generator['test']('foobarclient', ['test_main'],
                                           runTest = False, netTest = True))
         self.addTest(mm.generator['test']('foobarserver', ['test_main'],
@@ -5328,7 +5329,5 @@ def load(mm):
 
     mm.addModule(tests(mm))
 
-    mm.setModuleConfigiuration()
-
     # XXX TODO Check that no file is also listed in empty
     # XXX TODO Check that no file in in two modules

waf_libbsd.py

@@ -46,6 +46,10 @@ import builder
 
 import rtems_waf.rtems as rtems
 
+
+BUILDSET_DIR = builder.BUILDSET_DIR
+BUILDSET_DEFAULT = builder.BUILDSET_DEFAULT
+
 windows = os.name == 'nt'
 
 if windows:
@@ -138,16 +142,22 @@ class Builder(builder.ModuleManager):
 
         self.data = {}
 
-        for mn in self.getEnabledModules():
+        enabled_modules = self.getEnabledModules()
+        for mn in enabled_modules:
             m = self[mn]
-            if m.conditionalOn == "none":
+            enabled = True
+            for dep in m.dependencies:
+                if dep not in enabled_modules:
+                    enabled = False
+                    break
+            if enabled:
                 for f in m.files:
                     _dataInsert(self.data, 'all', f.getSpace(),
                                 f.getFragment())
-            for cpu, files in sorted(m.cpuDependentSourceFiles.items()):
-                for f in files:
-                    _dataInsert(self.data, cpu, f.getSpace(),
-                                f.getFragment())
+                for cpu, files in sorted(m.cpuDependentSourceFiles.items()):
+                    for f in files:
+                        _dataInsert(self.data, cpu, f.getSpace(),
+                                    f.getFragment())
 
         # Start here if you need to understand self.data. Add 'True or'
         if self.trace:

wscript

@@ -45,76 +45,24 @@ except:
     import sys
     sys.exit(1)
 
-import libbsd
-import waf_libbsd
 import os.path
 import runpy
 import sys
-try:
-    import configparser
-except ImportError:
-    import ConfigParser as configparser
+
 import waflib.Options
 
+import libbsd
+import waf_libbsd
+
 builders = {}
 
-BUILDSET_DIR = "buildset"
-BUILDSET_DEFAULT = "buildset/default.ini"
-
-
-def load_ini(conf, f):
-    ini = configparser.ConfigParser()
-    ini.read(f)
-    if not ini.has_section('general'):
-        conf.fatal("'{}' is missing a general section.".format(f))
-    if not ini.has_option('general', 'name'):
-        conf.fatal("'{}' is missing a general/name.".format(f))
-    if ini.has_option('general', 'extends'):
-        extends = ini.get('general', 'extends')
-        extendfile = None
-        basepath = os.path.dirname(f)
-        if os.path.isfile(os.path.join(basepath, extends)):
-            extendfile = os.path.join(basepath, extends)
-        elif os.path.isfile(os.path.join(BUILDSET_DIR, extends)):
-            extendfile = os.path.join(BUILDSET_DIR, extends)
-        else:
-            conf.fatal(
-                "'{}': Invalid file given for general/extends:'{}'".format(
-                    f, extends))
-        base = load_ini(conf, extendfile)
-        for s in ini.sections():
-            if not base.has_section(s):
-                base.add_section(s)
-            for o in ini.options(s):
-                val = ini.get(s, o)
-                base.set(s, o, val)
-        ini = base
-    return ini
-
-
-def load_config(conf, f):
-    ini = load_ini(conf, f)
-    config = {}
-
-    config['name'] = ini.get('general', 'name')
-
-    config['modules-enabled'] = []
-    mods = []
-    if ini.has_section('modules'):
-        mods = ini.options('modules')
-    for mod in mods:
-        if ini.getboolean('modules', mod):
-            config['modules-enabled'].append(mod)
-    return config
-
 
 def update_builders(ctx, buildset_opt):
     global builders
     builders = {}
-
     buildsets = []
     if buildset_opt == []:
-        buildset_opt.append(BUILDSET_DEFAULT)
+        buildset_opt.append(waf_libbsd.BUILDSET_DEFAULT)
     for bs in buildset_opt:
         if os.path.isdir(bs):
             for f in os.listdir(bs):
@@ -123,15 +71,16 @@ def update_builders(ctx, buildset_opt):
         else:
             for f in bs.split(','):
                 buildsets += [f]
-
     for bs in buildsets:
-        builder = waf_libbsd.Builder()
-        libbsd.load(builder)
-        bsconfig = load_config(ctx, bs)
-        bsname = bsconfig['name']
-        builder.updateConfiguration(bsconfig)
-        builder.generate(rtems_version)
-        builders[bsname] = builder
+        try:
+            builder = waf_libbsd.Builder()
+            libbsd.load(builder)
+            builder.loadConfig(bs)
+            builder.generate(rtems_version)
+        except Exception as exc:
+            raise
+            ctx.fatal(str(exc))
+        builders[builder.getName()] = builder
 
 
 def bsp_init(ctx, env, contexts):
@@ -250,7 +199,7 @@ def configure(conf):
     conf.env.OPTIMIZATION = conf.options.optimization
     conf.env.BUILDSET = conf.options.buildset
     if len(conf.env.BUILDSET) == 0:
-        conf.env.BUILDSET += [BUILDSET_DEFAULT]
+        conf.env.BUILDSET += [waf_libbsd.BUILDSET_DEFAULT]
     update_builders(conf, conf.env.BUILDSET)
     rtems.configure(conf, bsp_configure)