PR 2108, PR 2109. Add --jobs and --always-clean.

Refactor the options handling in defaults.py to allow the --jobs
option have varing specific parameters. The option supports 'none',
'max' and 'half' or a fraction to divide the number of CPUs or
an integer value which is the number of jobs. The --no-smp has
been removed.

The host specific modules have been changed to set the number of
CPUs in the defaults table.

Fixed the --keep-going to clean up is --always-clean is provided
even if the build has an error.

source-builder/sb/build.py

@@ -395,10 +395,7 @@ class build:
     def files(self, package):
         if self.create_tar_files:
             self.script.append('echo "==> %files:"')
-            prefixbase = self.opts.prefixbase()
+            inpath = path.abspath(self.config.expand('%{buildroot}'))
-            if prefixbase is None:
-                prefixbase = ''
-            inpath = path.join('%{buildroot}', prefixbase)
             tardir = path.abspath(self.config.expand('%{_tardir}'))
             self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
             self.script.append(self.config.expand('  %%{__mkdir_p} %s' % tardir))

source-builder/sb/darwin.py

@@ -33,10 +33,11 @@ def load():
     e = execute.capture_execution()
     exit_code, proc, output = e.shell(sysctl + 'hw.ncpu')
     if exit_code == 0:
-        smp_mflags = '-j' + output.split(' ')[1].strip()
+        ncpus = output.split(' ')[1].strip()
     else:
-        smp_mflags = ''
+        ncpus = '1'
     defines = {
+        '_ncpus':       ('none',    'none',     ncpus),
         '_os':          ('none',    'none',     'darwin'),
         '_host':        ('triplet', 'required', uname[4] + '-apple-darwin' + uname[2]),
         '_host_vendor': ('none',    'none',     'apple'),
@@ -48,7 +49,6 @@ def load():
         '_var':         ('dir',     'optional', '/usr/local/var'),
         '_prefix':      ('dir',     'optional', '%{_usr}'),
         'optflags':     ('none',    'none',     '-O2'),
-        '_smp_mflags':  ('none',    'none',     smp_mflags),
         '__ldconfig':   ('exe',     'none',     ''),
         '__xz':         ('exe',     'required', '%{_usr}/bin/xz'),
         'with_zlib':    ('none',    'none',     '--with-zlib=no')

source-builder/sb/defaults.py

@@ -251,68 +251,35 @@ FFLAGS="${FFLAGS:-%optflags}" ; export FFLAGS ;
 class command_line:
     """Process the command line in a common way for all Tool Builder commands."""
 
-    #
-    # The define and if it is a path and needs conversion.
-    #
-    _long_opts = { '--prefix'         : ('_prefix', True),
-                   '--prefixbase'     : ('_prefixbase', True),
-                   '--topdir'         : ('_topdir', True),
-                   '--configdir'      : ('_configdir', True),
-                   '--builddir'       : ('_builddir', True),
-                   '--sourcedir'      : ('_sourcedir', True),
-                   '--tmppath'        : ('_tmppath', True),
-                   '--log'            : ('_logfile', False),
-                   '--url'            : ('_url_base', False),
-                   '--targetcflags'   : ('_targetcflags', False),
-                   '--targetcxxflags' : ('_targetcxxflags', False),
-                   '--libstdcxxflags' : ('_libstdcxxflags', False) }
-
-    _long_true_opts = { '--force'    : '_force',
-                        '--trace'    : '_trace',
-                        '--dry-run'  : '_dry_run',
-                        '--warn-all' : '_warn_all',
-                        '--no-clean' : '_no_clean',
-                        '--no-smp'   : '_no_smp',
-                        '--rebuild'  : '_rebuild' }
-
-    _target_triplets = { '--host'   : '_host',
-                         '--build'  : '_build',
-                         '--target' : '_target' }
-
-    def _help(self):
-        print '%s: [options] [args]' % (self.command_name)
-        print 'RTEMS Source Builder, an RTEMS Tools Project (c) 2012-2013 Chris Johns'
-        print 'Options and arguments:'
-        print '--force                : Force the build to proceed'
-        print '--trace                : Trace the execution (not current used)'
-        print '--dry-run              : Do everything but actually run the build'
-        print '--warn-all             : Generate warnings'
-        print '--no-clean             : Do not clean up the build tree'
-        print '--no-smp               : Run with 1 job and not as many as CPUs'
-        print '--rebuild              : Rebuild (not used)'
-        print '--host                 : Set the host triplet'
-        print '--build                : Set the build triplet'
-        print '--target               : Set the target triplet'
-        print '--prefix path          : Tools build prefix, ie where they are installed'
-        print '--prefixbase path      : '
-        print '--topdir path          : Top of the build tree, default is $PWD'
-        print '--configdir path       : Path to the configuration directory, default: ./config'
-        print '--builddir path        : Path to the build directory, default: ./build'
-        print '--sourcedir path       : Path to the source directory, default: ./source'
-        print '--tmppath path         : Path to the temp directory, default: ./tmp'
-        print '--log file             : Log file where all build out is written too'
-        print '--url url              : URL to look for source'
-        print '--targetcflags flags   : List of C flags for the target code'
-        print '--targetcxxflags flags : List of C++ flags for the target code'
-        print '--libstdcxxflags flags : List of C++ flags to build the target libstdc++ code'
-        print '--with-<label>         : Add the --with-<label> to the build'
-        print '--without-<label>      : Add the --without-<label> to the build'
-        if self.optargs:
-            for a in self.optargs:
-                print '%-22s : %s' % (a, self.optargs[a])
-        raise error.exit()
-
     def __init__(self, argv, optargs):
+        self._long_opts = {
+            # key                 macro              handler            param  defs    init
+            '--prefix'         : ('_prefix',         self._lo_path,     True,  None,  False),
+            '--topdir'         : ('_topdir',         self._lo_path,     True,  None,  False),
+            '--configdir'      : ('_configdir',      self._lo_path,     True,  None,  False),
+            '--builddir'       : ('_builddir',       self._lo_path,     True,  None,  False),
+            '--sourcedir'      : ('_sourcedir',      self._lo_path,     True,  None,  False),
+            '--tmppath'        : ('_tmppath',        self._lo_path,     True,  None,  False),
+            '--jobs'           : ('_jobs',           self._lo_jobs,     True,  'max', True),
+            '--log'            : ('_logfile',        self._lo_string,   True,  None,  False),
+            '--url'            : ('_url_base',       self._lo_string,   True,  None,  False),
+            '--targetcflags'   : ('_targetcflags',   self._lo_string,   True,  None,  False),
+            '--targetcxxflags' : ('_targetcxxflags', self._lo_string,   True,  None,  False),
+            '--libstdcxxflags' : ('_libstdcxxflags', self._lo_string,   True,  None,  False),
+            '--force'          : ('_force',          self._lo_bool,     False, '0',   True),
+            '--quiet'          : ('_quiet',          self._lo_bool,     False, '0',   True),
+            '--trace'          : ('_trace',          self._lo_bool,     False, '0',   True),
+            '--dry-run'        : ('_dry_run',        self._lo_bool,     False, '0',   True),
+            '--warn-all'       : ('_warn_all',       self._lo_bool,     False, '0',   True),
+            '--no-clean'       : ('_no_clean',       self._lo_bool,     False, '0',   True),
+            '--keep-going'     : ('_keep_going',     self._lo_bool,     False, '0',   True),
+            '--always-clean'   : ('_always_clean',   self._lo_bool,     False, '0',   True),
+            '--host'           : ('_host',           self._lo_triplets, True,  None,  False),
+            '--build'          : ('_build',          self._lo_triplets, True,  None,  False),
+            '--target'         : ('_target',         self._lo_triplets, True,  None,  False),
+            '--help'           : (None,              self._lo_help,     False, None,  False)
+            }
+
         self.command_path = path.dirname(argv[0])
         if len(self.command_path) == 0:
             self.command_path = '.'
@@ -321,18 +288,12 @@ class command_line:
         self.args = argv[1:]
         self.optargs = optargs
         self.defaults = {}
-        for to in command_line._long_true_opts:
-            self.defaults[command_line._long_true_opts[to]] = ('none', 'none', '0')
         self.defaults['_sbdir'] = ('dir', 'required', path.shell(self.command_path))
-        self.opts = { 'params'   : [],
+        self.opts = { 'params' : [] }
-                      'warn-all' : '0',
+        for lo in self._long_opts:
-                      'quiet'    : '0',
+            self.opts[lo[2:]] = self._long_opts[lo][3]
-                      'force'    : '0',
+            if self._long_opts[lo][4]:
-                      'trace'    : '0',
+                self.defaults[self._long_opts[lo][0]] = ('none', 'none', self._long_opts[lo][3])
-                      'dry-run'  : '0',
-                      'no-clean' : '0',
-                      'no-smp'   : '0',
-                      'rebuild'  : '0' }
         self._process()
 
     def __str__(self):
@@ -350,64 +311,52 @@ class command_line:
 
         return s
 
-    def _process(self):
+    def _lo_string(self, opt, macro, value):
-
+        if value is None:
-        def _process_lopt(opt, arg, long_opts, args, values = False):
+            raise error.general('option requires a value: %s' % (opt))
-            for lo in long_opts:
+        self.opts[opt[2:]] = value
-                if values and opt.startswith(lo):
-                    equals = opt.find('=')
-                    if equals < 0:
-                        if arg == len(args) - 1:
-                            raise error.general('missing option value: ' + lo)
-                        arg += 1
-                        value = args[arg]
-                    else:
-                        value = opt[equals + 1:]
-                    if type(long_opts[lo]) is tuple:
-                        if long_opts[lo][1]:
-                            value = path.shell(value)
-                        macro = long_opts[lo][0]
-                    else:
-                        macro = long_opts[lo]
-                    return lo, macro, value, arg
-                elif opt == lo:
-                    return lo, long_opts[lo], True, arg
-            return None, None, None, arg
-
-        i = 0
-        while i < len(self.args):
-            a = self.args[i]
-            if a.startswith('-'):
-                if a.startswith('--'):
-                    if a.startswith('--warn-all'):
-                        self.opts['warn-all'] = True
-                    elif a == '--help':
-                        self._help()
-                    else:
-                        lo, macro, value, i = _process_lopt(a, i,
-                                                            command_line._long_true_opts,
-                                                            self.args)
-                        if lo:
-                            self.defaults[macro] = ('none', 'none', '1')
-                            self.opts[lo[2:]] = '1'
-                        else:
-                            lo, macro, value, i = _process_lopt(a, i,
-                                                                command_line._long_opts,
-                                                                self.args, True)
-                            if lo:
         self.defaults[macro] = ('none', 'none', value)
-                                self.opts[lo[2:]] = value
+
+    def _lo_path(self, opt, macro, value):
+        if value is None:
+            raise error.general('option requires a path: %s' % (opt))
+        value = path.shell(value)
+        self.opts[opt[2:]] = value
+        self.defaults[macro] = ('none', 'none', value)
+
+    def _lo_jobs(self, opt, macro, value):
+        if value is None:
+            raise error.general('option requires a value: %s' % (opt))
+        ok = False
+        if value in ['max', 'none', 'half']:
+            ok = True
         else:
+            try:
+                i = int(value)
+                ok = True
+            except:
+                pass
+            if not ok:
+                try:
+                    f = float(value)
+                    ok = True
+                except:
+                    pass
+        if not ok:
+            raise error.general('invalid jobs option: %s' % (value))
+        self.defaults[macro] = ('none', 'none', value)
+        self.opts[opt[2:]] = value
+
+    def _lo_bool(self, opt, macro, value):
+        if value is not None:
+            raise error.general('option does not take a value: %s' % (opt))
+        self.opts[opt[2:]] = '1'
+        self.defaults[macro] = ('none', 'none', '1')
+
+    def _lo_triplets(self, opt, macro, value):
         #
-                                # The target triplet is 'cpu-vendor-os'.
+        # This is a target triplet. Run it past config.sub to make make sure it
-                                #
+        # is ok.  The target triplet is 'cpu-vendor-os'.
-                                lo, macro, value, i = _process_lopt(a, i,
-                                                                    command_line._target_triplets,
-                                                                    self.args, True)
-                                if lo:
-                                    #
-                                    # This is a target triplet. Run it past config.sub to make
-                                    # make sure it is ok.
         #
         e = execute.capture_execution()
         config_sub = path.join(self.command_path,
@@ -416,8 +365,9 @@ class command_line:
         if exit_code == 0:
             value = output
         self.defaults[macro] = ('triplet', 'none', value)
-                                    self.opts[lo[2:]] = value
+        self.opts[opt[2:]] = value
-                                    _arch = macro + '_cpu'
+        _cpu = macro + '_cpu'
+        _arch = macro + '_arch'
         _vendor = macro + '_vendor'
         _os = macro + '_os'
         _arch_value = ''
@@ -433,33 +383,83 @@ class command_line:
             value = value[dash + 1:]
         if len(value):
             _os_value = value
+        self.defaults[_cpu] = ('none', 'none', _arch_value)
         self.defaults[_arch] = ('none', 'none', _arch_value)
         self.defaults[_vendor] = ('none', 'none', _vendor_value)
         self.defaults[_os] = ('none', 'none', _os_value)
-                                if not lo:
+
-                                    sa = a.split('=')
+    def _lo_help(self, opt, macro, value):
-                                    if sa[0] not in self.optargs:
+        self.help()
-                                        raise error.general('invalid argument (try --help): %s' % (a))
+
-                else:
+    def _help(self):
-                    if a == '-f':
+        print '%s: [options] [args]' % (self.command_name)
-                        self.opts['force'] = '1'
+        print 'RTEMS Source Builder, an RTEMS Tools Project (c) 2012-2013 Chris Johns'
-                    elif a == '-n':
+        print 'Options and arguments:'
-                        self.opts['dry-run'] = '1'
+        print '--force                : Force the build to proceed'
-                    elif a == '-q':
+        print '--quiet                : Quiet output (not used)'
-                        self.opts['quiet'] = '1'
+        print '--trace                : Trace the execution'
-                    elif a == '-?':
+        print '--dry-run              : Do everything but actually run the build'
+        print '--warn-all             : Generate warnings'
+        print '--no-clean             : Do not clean up the build tree'
+        print '--always-clean         : Always clean the build tree, even with an error'
+        print '--jobs                 : Run with specified number of jobs, default: num CPUs.'
+        print '--host                 : Set the host triplet'
+        print '--build                : Set the build triplet'
+        print '--target               : Set the target triplet'
+        print '--prefix path          : Tools build prefix, ie where they are installed'
+        print '--topdir path          : Top of the build tree, default is $PWD'
+        print '--configdir path       : Path to the configuration directory, default: ./config'
+        print '--builddir path        : Path to the build directory, default: ./build'
+        print '--sourcedir path       : Path to the source directory, default: ./source'
+        print '--tmppath path         : Path to the temp directory, default: ./tmp'
+        print '--log file             : Log file where all build out is written too'
+        print '--url url              : URL to look for source'
+        print '--targetcflags flags   : List of C flags for the target code'
+        print '--targetcxxflags flags : List of C++ flags for the target code'
+        print '--libstdcxxflags flags : List of C++ flags to build the target libstdc++ code'
+        print '--with-<label>         : Add the --with-<label> to the build'
+        print '--without-<label>      : Add the --without-<label> to the build'
+        if self.optargs:
+            for a in self.optargs:
+                print '%-22s : %s' % (a, self.optargs[a])
+        raise error.exit()
+
+    def _process(self):
+        arg = 0
+        while arg < len(self.args):
+            a = self.args[arg]
+            if a == '-?':
                 self._help()
+            elif a.startswith('--'):
+                los = a.split('=')
+                lo = los[0]
+                if lo in self._long_opts:
+                    long_opt = self._long_opts[lo]
+                    if len(los) == 1:
+                        if long_opt[2]:
+                            if arg == len(args) - 1:
+                                raise error.general('option requires a parameter: %s' % (lo))
+                            arg += 1
+                            value = args[arg]
                         else:
-                        raise error.general('invalid argument (try --help): %s' % (a))
+                            value = None
+                    else:
+                        value = '='.join(los[1:])
+                    long_opt[1](lo, long_opt[0], value)
             else:
                 self.opts['params'].append(a)
-            i += 1
+            arg += 1
 
     def _post_process(self, _defaults):
-        if self.no_smp():
-            _defaults['_smp_mflags'] = ('none', 'none', _defaults['nil'][2])
         if _defaults['_host'][2] == _defaults['nil'][2]:
             raise error.general('host not set')
+        if '_ncpus' not in _defaults:
+            raise error.general('host number of CPUs not set')
+        ncpus = self.jobs(_defaults['_ncpus'][2])
+        if ncpus > 1:
+            _defaults['_smp_mflags'] = ('none', 'none', '-j %d' % (ncpus))
+        else:
+            _defaults['_smp_mflags'] = ('none', 'none', _defaults['nil'][2])
         return _defaults
 
     def define(self, _defaults, key, value = '1'):
@@ -506,14 +506,43 @@ class command_line:
     def warn_all(self):
         return self.opts['warn-all'] != '0'
 
+    def keep_going(self):
+        return self.opts['keep-going'] != '0'
+
     def no_clean(self):
         return self.opts['no-clean'] != '0'
 
-    def no_smp(self):
+    def always_clean(self):
-        return self.opts['no-smp'] != '0'
+        return self.opts['always-clean'] != '0'
 
-    def rebuild(self):
+    def jobs(self, cpus):
-        return self.opts['rebuild'] != '0'
+        cpus = int(cpus)
+        if self.opts['jobs'] == 'none':
+            cpus = 0
+        elif self.opts['jobs'] == 'max':
+            pass
+        elif self.opts['jobs'] == 'half':
+            cpus = cpus / 2
+        else:
+            ok = False
+            try:
+                i = int(self.opts['jobs'])
+                cpus = i
+                ok = True
+            except:
+                pass
+            if not ok:
+                try:
+                    f = float(self.opts['jobs'])
+                    cpus = f * cpus
+                    ok = True
+                except:
+                    pass
+                if not ok:
+                    raise error.internal('bad jobs option: %s' % (self.opts['jobs']))
+        if cpus <= 0:
+            cpu = 1
+        return cpus
 
     def params(self):
         return self.opts['params']
@@ -566,15 +595,10 @@ class command_line:
         return ['stdout']
 
     def urls(self):
-        if 'url' in self.opts:
+        if self.opts['url'] is not None:
             return self.opts['url'].split(',')
         return None
 
-    def prefixbase(self):
-        if 'prefixbase' in self.opts:
-            return self.opts['prefixbase']
-        return None
-
 def load(args, optargs = None):
     """
     Copy the defaults, get the host specific values and merge them overriding

source-builder/sb/freebsd.py

@@ -35,9 +35,9 @@ def load():
     e = execute.capture_execution()
     exit_code, proc, output = e.shell(sysctl + 'hw.ncpu')
     if exit_code == 0:
-        smp_mflags = '-j' + output.split(' ')[1].strip()
+        ncpus = output.split(' ')[1].strip()
     else:
-        smp_mflags = ''
+        ncpus = '1'
     if uname[4] == 'amd64':
         cpu = 'x86_64'
     else:
@@ -46,6 +46,7 @@ def load():
     if version.find('-') > 0:
         version = version.split('-')[0]
     defines = {
+        '_ncpus':       ('none',    'none',     ncpus),
         '_os':          ('none',    'none',     'freebsd'),
         '_host':        ('triplet', 'required', cpu + '-freebsd' + version),
         '_host_vendor': ('none',    'none',     'pc'),
@@ -56,7 +57,6 @@ def load():
         '_usr':         ('dir',     'required', '/usr/local'),
         '_var':         ('dir',     'optional', '/usr/local/var'),
         'optflags':     ('none',    'none',     '-O2 -I/usr/local/include -L/usr/local/lib'),
-        '_smp_mflags':  ('none',    'none',     smp_mflags),
         '__bash':       ('exe',     'optional', '/usr/local/bin/bash'),
         '__bison':      ('exe',     'required', '/usr/local/bin/bison'),
         '__git':        ('exe',     'required', '/usr/local/bin/git'),

source-builder/sb/linux.py

@@ -35,23 +35,23 @@ def load():
     processors = '/bin/grep processor /proc/cpuinfo'
     e = execute.capture_execution()
     exit_code, proc, output = e.shell(processors)
+    ncpus = 0
     if exit_code == 0:
-        cpus = 0
         try:
             for l in output.split('\n'):
                 count = l.split(':')[1].strip()
                 if count > cpus:
-                    cpus = int(count)
+                    ncpus = int(count)
-            if cpus > 0:
-                smp_mflags = '-j%d' % (cpus + 1)
         except:
             pass
+    ncpus = str(ncpus + 1)
     if uname[4].startswith('arm'):
         cpu = 'arm'
     else:
         cpu = uname[4]
 
     defines = {
+        '_ncpus':         ('none',    'none',     ncpus),
         '_os':            ('none',    'none',     'linux'),
         '_host':          ('triplet', 'required', cpu + '-linux-gnu'),
         '_host_vendor':   ('none',    'none',     'gnu'),
@@ -62,7 +62,6 @@ def load():
         '_usr':           ('dir',     'required', '/usr'),
         '_var':           ('dir',     'required', '/var'),
         'optflags':       ('none',    'none',     '-O2 -pipe'),
-        '_smp_mflags':    ('none',    'none',     smp_mflags),
         '__bzip2':        ('exe',     'required', '/usr/bin/bzip2'),
         '__gzip':         ('exe',     'required', '/bin/gzip'),
         '__tar':          ('exe',     'required', '/bin/tar')

source-builder/sb/setbuilder.py

@@ -311,8 +311,10 @@ class buildset:
                     else:
                         raise error.general('invalid config type: %s' % (configs[s]))
                 except error.general, gerr:
-                    if self.opts.get_arg('--keep-going'):
+                    if self.opts.keep_going():
                         print gerr
+                        if self.opts.always_clean():
+                            builds += [b]
                     else:
                         raise
             if deps is None and not self.opts.get_arg('--no-install'):
@@ -321,7 +323,7 @@ class buildset:
                                  b.config.expand('%{buildroot}'),
                                  b.config.expand('%{_prefix}'))
             if deps is None and \
-                    (not self.opts.no_clean() or self.opts.get_arg('--keep-going')):
+                    (not self.opts.no_clean() or self.opts.always_clean()):
                 for b in builds:
                     _notice(self.opts, 'cleaning: %s' % (b.name()))
                     b.cleanup()
@@ -357,7 +359,6 @@ def run():
         optargs = { '--list-configs':  'List available configurations',
                     '--list-bsets':    'List available build sets',
                     '--list-deps':     'List the dependent files.',
-                    '--keep-going':    'Do not stop on error.',
                     '--no-install':    'Do not install the packages to the prefix.',
                     '--no-report':     'Do not create a package report.',
                     '--report-format': 'The report format (text, html, asciidoc).',

source-builder/sb/windows.py

@@ -58,15 +58,12 @@ def load():
             pass
 
     if os.environ.has_key('NUMBER_OF_PROCESSORS'):
-        ncpus = int(os.environ['NUMBER_OF_PROCESSORS'])
+        ncpus = os.environ['NUMBER_OF_PROCESSORS']
     else:
-        ncpus = 0
+        ncpus = '1'
-    if ncpus > 1:
-        smp_mflags = '-j' + str(ncpus)
-    else:
-        smp_mflags = ''
 
     defines = {
+        '_ncpus':         ('none',    'none',     ncpus),
         '_os':            ('none',    'none',     'win32'),
         '_build':         ('triplet', 'required', build_triple),
         '_host':          ('triplet', 'required', host_triple),