PR 2108, PR 2109. Add --jobs and --always-clean.

Refactor the options handling in defaults.py to allow the --jobs
option have varing specific parameters. The option supports 'none',
'max' and 'half' or a fraction to divide the number of CPUs or
an integer value which is the number of jobs. The --no-smp has
been removed.

The host specific modules have been changed to set the number of
CPUs in the defaults table.

Fixed the --keep-going to clean up is --always-clean is provided
even if the build has an error.
This commit is contained in:
Chris Johns 2013-04-03 14:31:41 +11:00
parent e4cb1d01d6
commit 0add2eaa70
7 changed files with 231 additions and 213 deletions

View File

@ -395,10 +395,7 @@ class build:
def files(self, package):
if self.create_tar_files:
self.script.append('echo "==> %files:"')
prefixbase = self.opts.prefixbase()
if prefixbase is None:
prefixbase = ''
inpath = path.join('%{buildroot}', prefixbase)
inpath = path.abspath(self.config.expand('%{buildroot}'))
tardir = path.abspath(self.config.expand('%{_tardir}'))
self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
self.script.append(self.config.expand(' %%{__mkdir_p} %s' % tardir))
@ -521,7 +518,7 @@ def run(args):
_notice(opts, 'RTEMS Source Builder, Package Builder v%s' % (version))
if not check.host_setup(opts, _defaults):
if not opts.force():
raise error.general('host build environment is not set up' +
raise error.general('host build environment is not set up' +
' correctly (use --force to proceed)')
_notice(opts, 'warning: forcing build with known host setup problems')
if opts.get_arg('--list-configs'):

View File

@ -33,10 +33,11 @@ def load():
e = execute.capture_execution()
exit_code, proc, output = e.shell(sysctl + 'hw.ncpu')
if exit_code == 0:
smp_mflags = '-j' + output.split(' ')[1].strip()
ncpus = output.split(' ')[1].strip()
else:
smp_mflags = ''
ncpus = '1'
defines = {
'_ncpus': ('none', 'none', ncpus),
'_os': ('none', 'none', 'darwin'),
'_host': ('triplet', 'required', uname[4] + '-apple-darwin' + uname[2]),
'_host_vendor': ('none', 'none', 'apple'),
@ -48,7 +49,6 @@ def load():
'_var': ('dir', 'optional', '/usr/local/var'),
'_prefix': ('dir', 'optional', '%{_usr}'),
'optflags': ('none', 'none', '-O2'),
'_smp_mflags': ('none', 'none', smp_mflags),
'__ldconfig': ('exe', 'none', ''),
'__xz': ('exe', 'required', '%{_usr}/bin/xz'),
'with_zlib': ('none', 'none', '--with-zlib=no')

View File

@ -251,68 +251,35 @@ FFLAGS="${FFLAGS:-%optflags}" ; export FFLAGS ;
class command_line:
"""Process the command line in a common way for all Tool Builder commands."""
#
# The define and if it is a path and needs conversion.
#
_long_opts = { '--prefix' : ('_prefix', True),
'--prefixbase' : ('_prefixbase', True),
'--topdir' : ('_topdir', True),
'--configdir' : ('_configdir', True),
'--builddir' : ('_builddir', True),
'--sourcedir' : ('_sourcedir', True),
'--tmppath' : ('_tmppath', True),
'--log' : ('_logfile', False),
'--url' : ('_url_base', False),
'--targetcflags' : ('_targetcflags', False),
'--targetcxxflags' : ('_targetcxxflags', False),
'--libstdcxxflags' : ('_libstdcxxflags', False) }
_long_true_opts = { '--force' : '_force',
'--trace' : '_trace',
'--dry-run' : '_dry_run',
'--warn-all' : '_warn_all',
'--no-clean' : '_no_clean',
'--no-smp' : '_no_smp',
'--rebuild' : '_rebuild' }
_target_triplets = { '--host' : '_host',
'--build' : '_build',
'--target' : '_target' }
def _help(self):
print '%s: [options] [args]' % (self.command_name)
print 'RTEMS Source Builder, an RTEMS Tools Project (c) 2012-2013 Chris Johns'
print 'Options and arguments:'
print '--force : Force the build to proceed'
print '--trace : Trace the execution (not current used)'
print '--dry-run : Do everything but actually run the build'
print '--warn-all : Generate warnings'
print '--no-clean : Do not clean up the build tree'
print '--no-smp : Run with 1 job and not as many as CPUs'
print '--rebuild : Rebuild (not used)'
print '--host : Set the host triplet'
print '--build : Set the build triplet'
print '--target : Set the target triplet'
print '--prefix path : Tools build prefix, ie where they are installed'
print '--prefixbase path : '
print '--topdir path : Top of the build tree, default is $PWD'
print '--configdir path : Path to the configuration directory, default: ./config'
print '--builddir path : Path to the build directory, default: ./build'
print '--sourcedir path : Path to the source directory, default: ./source'
print '--tmppath path : Path to the temp directory, default: ./tmp'
print '--log file : Log file where all build out is written too'
print '--url url : URL to look for source'
print '--targetcflags flags : List of C flags for the target code'
print '--targetcxxflags flags : List of C++ flags for the target code'
print '--libstdcxxflags flags : List of C++ flags to build the target libstdc++ code'
print '--with-<label> : Add the --with-<label> to the build'
print '--without-<label> : Add the --without-<label> to the build'
if self.optargs:
for a in self.optargs:
print '%-22s : %s' % (a, self.optargs[a])
raise error.exit()
def __init__(self, argv, optargs):
self._long_opts = {
# key macro handler param defs init
'--prefix' : ('_prefix', self._lo_path, True, None, False),
'--topdir' : ('_topdir', self._lo_path, True, None, False),
'--configdir' : ('_configdir', self._lo_path, True, None, False),
'--builddir' : ('_builddir', self._lo_path, True, None, False),
'--sourcedir' : ('_sourcedir', self._lo_path, True, None, False),
'--tmppath' : ('_tmppath', self._lo_path, True, None, False),
'--jobs' : ('_jobs', self._lo_jobs, True, 'max', True),
'--log' : ('_logfile', self._lo_string, True, None, False),
'--url' : ('_url_base', self._lo_string, True, None, False),
'--targetcflags' : ('_targetcflags', self._lo_string, True, None, False),
'--targetcxxflags' : ('_targetcxxflags', self._lo_string, True, None, False),
'--libstdcxxflags' : ('_libstdcxxflags', self._lo_string, True, None, False),
'--force' : ('_force', self._lo_bool, False, '0', True),
'--quiet' : ('_quiet', self._lo_bool, False, '0', True),
'--trace' : ('_trace', self._lo_bool, False, '0', True),
'--dry-run' : ('_dry_run', self._lo_bool, False, '0', True),
'--warn-all' : ('_warn_all', self._lo_bool, False, '0', True),
'--no-clean' : ('_no_clean', self._lo_bool, False, '0', True),
'--keep-going' : ('_keep_going', self._lo_bool, False, '0', True),
'--always-clean' : ('_always_clean', self._lo_bool, False, '0', True),
'--host' : ('_host', self._lo_triplets, True, None, False),
'--build' : ('_build', self._lo_triplets, True, None, False),
'--target' : ('_target', self._lo_triplets, True, None, False),
'--help' : (None, self._lo_help, False, None, False)
}
self.command_path = path.dirname(argv[0])
if len(self.command_path) == 0:
self.command_path = '.'
@ -321,18 +288,12 @@ class command_line:
self.args = argv[1:]
self.optargs = optargs
self.defaults = {}
for to in command_line._long_true_opts:
self.defaults[command_line._long_true_opts[to]] = ('none', 'none', '0')
self.defaults['_sbdir'] = ('dir', 'required', path.shell(self.command_path))
self.opts = { 'params' : [],
'warn-all' : '0',
'quiet' : '0',
'force' : '0',
'trace' : '0',
'dry-run' : '0',
'no-clean' : '0',
'no-smp' : '0',
'rebuild' : '0' }
self.opts = { 'params' : [] }
for lo in self._long_opts:
self.opts[lo[2:]] = self._long_opts[lo][3]
if self._long_opts[lo][4]:
self.defaults[self._long_opts[lo][0]] = ('none', 'none', self._long_opts[lo][3])
self._process()
def __str__(self):
@ -350,116 +311,155 @@ class command_line:
return s
def _lo_string(self, opt, macro, value):
if value is None:
raise error.general('option requires a value: %s' % (opt))
self.opts[opt[2:]] = value
self.defaults[macro] = ('none', 'none', value)
def _lo_path(self, opt, macro, value):
if value is None:
raise error.general('option requires a path: %s' % (opt))
value = path.shell(value)
self.opts[opt[2:]] = value
self.defaults[macro] = ('none', 'none', value)
def _lo_jobs(self, opt, macro, value):
if value is None:
raise error.general('option requires a value: %s' % (opt))
ok = False
if value in ['max', 'none', 'half']:
ok = True
else:
try:
i = int(value)
ok = True
except:
pass
if not ok:
try:
f = float(value)
ok = True
except:
pass
if not ok:
raise error.general('invalid jobs option: %s' % (value))
self.defaults[macro] = ('none', 'none', value)
self.opts[opt[2:]] = value
def _lo_bool(self, opt, macro, value):
if value is not None:
raise error.general('option does not take a value: %s' % (opt))
self.opts[opt[2:]] = '1'
self.defaults[macro] = ('none', 'none', '1')
def _lo_triplets(self, opt, macro, value):
#
# This is a target triplet. Run it past config.sub to make make sure it
# is ok. The target triplet is 'cpu-vendor-os'.
#
e = execute.capture_execution()
config_sub = path.join(self.command_path,
basepath, 'config.sub')
exit_code, proc, output = e.shell(config_sub + ' ' + value)
if exit_code == 0:
value = output
self.defaults[macro] = ('triplet', 'none', value)
self.opts[opt[2:]] = value
_cpu = macro + '_cpu'
_arch = macro + '_arch'
_vendor = macro + '_vendor'
_os = macro + '_os'
_arch_value = ''
_vendor_value = ''
_os_value = ''
dash = value.find('-')
if dash >= 0:
_arch_value = value[:dash]
value = value[dash + 1:]
dash = value.find('-')
if dash >= 0:
_vendor_value = value[:dash]
value = value[dash + 1:]
if len(value):
_os_value = value
self.defaults[_cpu] = ('none', 'none', _arch_value)
self.defaults[_arch] = ('none', 'none', _arch_value)
self.defaults[_vendor] = ('none', 'none', _vendor_value)
self.defaults[_os] = ('none', 'none', _os_value)
def _lo_help(self, opt, macro, value):
self.help()
def _help(self):
print '%s: [options] [args]' % (self.command_name)
print 'RTEMS Source Builder, an RTEMS Tools Project (c) 2012-2013 Chris Johns'
print 'Options and arguments:'
print '--force : Force the build to proceed'
print '--quiet : Quiet output (not used)'
print '--trace : Trace the execution'
print '--dry-run : Do everything but actually run the build'
print '--warn-all : Generate warnings'
print '--no-clean : Do not clean up the build tree'
print '--always-clean : Always clean the build tree, even with an error'
print '--jobs : Run with specified number of jobs, default: num CPUs.'
print '--host : Set the host triplet'
print '--build : Set the build triplet'
print '--target : Set the target triplet'
print '--prefix path : Tools build prefix, ie where they are installed'
print '--topdir path : Top of the build tree, default is $PWD'
print '--configdir path : Path to the configuration directory, default: ./config'
print '--builddir path : Path to the build directory, default: ./build'
print '--sourcedir path : Path to the source directory, default: ./source'
print '--tmppath path : Path to the temp directory, default: ./tmp'
print '--log file : Log file where all build out is written too'
print '--url url : URL to look for source'
print '--targetcflags flags : List of C flags for the target code'
print '--targetcxxflags flags : List of C++ flags for the target code'
print '--libstdcxxflags flags : List of C++ flags to build the target libstdc++ code'
print '--with-<label> : Add the --with-<label> to the build'
print '--without-<label> : Add the --without-<label> to the build'
if self.optargs:
for a in self.optargs:
print '%-22s : %s' % (a, self.optargs[a])
raise error.exit()
def _process(self):
def _process_lopt(opt, arg, long_opts, args, values = False):
for lo in long_opts:
if values and opt.startswith(lo):
equals = opt.find('=')
if equals < 0:
if arg == len(args) - 1:
raise error.general('missing option value: ' + lo)
arg += 1
value = args[arg]
else:
value = opt[equals + 1:]
if type(long_opts[lo]) is tuple:
if long_opts[lo][1]:
value = path.shell(value)
macro = long_opts[lo][0]
else:
macro = long_opts[lo]
return lo, macro, value, arg
elif opt == lo:
return lo, long_opts[lo], True, arg
return None, None, None, arg
i = 0
while i < len(self.args):
a = self.args[i]
if a.startswith('-'):
if a.startswith('--'):
if a.startswith('--warn-all'):
self.opts['warn-all'] = True
elif a == '--help':
self._help()
else:
lo, macro, value, i = _process_lopt(a, i,
command_line._long_true_opts,
self.args)
if lo:
self.defaults[macro] = ('none', 'none', '1')
self.opts[lo[2:]] = '1'
arg = 0
while arg < len(self.args):
a = self.args[arg]
if a == '-?':
self._help()
elif a.startswith('--'):
los = a.split('=')
lo = los[0]
if lo in self._long_opts:
long_opt = self._long_opts[lo]
if len(los) == 1:
if long_opt[2]:
if arg == len(args) - 1:
raise error.general('option requires a parameter: %s' % (lo))
arg += 1
value = args[arg]
else:
lo, macro, value, i = _process_lopt(a, i,
command_line._long_opts,
self.args, True)
if lo:
self.defaults[macro] = ('none', 'none', value)
self.opts[lo[2:]] = value
else:
#
# The target triplet is 'cpu-vendor-os'.
#
lo, macro, value, i = _process_lopt(a, i,
command_line._target_triplets,
self.args, True)
if lo:
#
# This is a target triplet. Run it past config.sub to make
# make sure it is ok.
#
e = execute.capture_execution()
config_sub = path.join(self.command_path,
basepath, 'config.sub')
exit_code, proc, output = e.shell(config_sub + ' ' + value)
if exit_code == 0:
value = output
self.defaults[macro] = ('triplet', 'none', value)
self.opts[lo[2:]] = value
_arch = macro + '_cpu'
_vendor = macro + '_vendor'
_os = macro + '_os'
_arch_value = ''
_vendor_value = ''
_os_value = ''
dash = value.find('-')
if dash >= 0:
_arch_value = value[:dash]
value = value[dash + 1:]
dash = value.find('-')
if dash >= 0:
_vendor_value = value[:dash]
value = value[dash + 1:]
if len(value):
_os_value = value
self.defaults[_arch] = ('none', 'none', _arch_value)
self.defaults[_vendor] = ('none', 'none', _vendor_value)
self.defaults[_os] = ('none', 'none', _os_value)
if not lo:
sa = a.split('=')
if sa[0] not in self.optargs:
raise error.general('invalid argument (try --help): %s' % (a))
else:
if a == '-f':
self.opts['force'] = '1'
elif a == '-n':
self.opts['dry-run'] = '1'
elif a == '-q':
self.opts['quiet'] = '1'
elif a == '-?':
self._help()
value = None
else:
raise error.general('invalid argument (try --help): %s' % (a))
value = '='.join(los[1:])
long_opt[1](lo, long_opt[0], value)
else:
self.opts['params'].append(a)
i += 1
arg += 1
def _post_process(self, _defaults):
if self.no_smp():
_defaults['_smp_mflags'] = ('none', 'none', _defaults['nil'][2])
if _defaults['_host'][2] == _defaults['nil'][2]:
raise error.general('host not set')
if '_ncpus' not in _defaults:
raise error.general('host number of CPUs not set')
ncpus = self.jobs(_defaults['_ncpus'][2])
if ncpus > 1:
_defaults['_smp_mflags'] = ('none', 'none', '-j %d' % (ncpus))
else:
_defaults['_smp_mflags'] = ('none', 'none', _defaults['nil'][2])
return _defaults
def define(self, _defaults, key, value = '1'):
@ -506,14 +506,43 @@ class command_line:
def warn_all(self):
return self.opts['warn-all'] != '0'
def keep_going(self):
return self.opts['keep-going'] != '0'
def no_clean(self):
return self.opts['no-clean'] != '0'
def no_smp(self):
return self.opts['no-smp'] != '0'
def always_clean(self):
return self.opts['always-clean'] != '0'
def rebuild(self):
return self.opts['rebuild'] != '0'
def jobs(self, cpus):
cpus = int(cpus)
if self.opts['jobs'] == 'none':
cpus = 0
elif self.opts['jobs'] == 'max':
pass
elif self.opts['jobs'] == 'half':
cpus = cpus / 2
else:
ok = False
try:
i = int(self.opts['jobs'])
cpus = i
ok = True
except:
pass
if not ok:
try:
f = float(self.opts['jobs'])
cpus = f * cpus
ok = True
except:
pass
if not ok:
raise error.internal('bad jobs option: %s' % (self.opts['jobs']))
if cpus <= 0:
cpu = 1
return cpus
def params(self):
return self.opts['params']
@ -566,15 +595,10 @@ class command_line:
return ['stdout']
def urls(self):
if 'url' in self.opts:
if self.opts['url'] is not None:
return self.opts['url'].split(',')
return None
def prefixbase(self):
if 'prefixbase' in self.opts:
return self.opts['prefixbase']
return None
def load(args, optargs = None):
"""
Copy the defaults, get the host specific values and merge them overriding

View File

@ -35,9 +35,9 @@ def load():
e = execute.capture_execution()
exit_code, proc, output = e.shell(sysctl + 'hw.ncpu')
if exit_code == 0:
smp_mflags = '-j' + output.split(' ')[1].strip()
ncpus = output.split(' ')[1].strip()
else:
smp_mflags = ''
ncpus = '1'
if uname[4] == 'amd64':
cpu = 'x86_64'
else:
@ -46,6 +46,7 @@ def load():
if version.find('-') > 0:
version = version.split('-')[0]
defines = {
'_ncpus': ('none', 'none', ncpus),
'_os': ('none', 'none', 'freebsd'),
'_host': ('triplet', 'required', cpu + '-freebsd' + version),
'_host_vendor': ('none', 'none', 'pc'),
@ -56,7 +57,6 @@ def load():
'_usr': ('dir', 'required', '/usr/local'),
'_var': ('dir', 'optional', '/usr/local/var'),
'optflags': ('none', 'none', '-O2 -I/usr/local/include -L/usr/local/lib'),
'_smp_mflags': ('none', 'none', smp_mflags),
'__bash': ('exe', 'optional', '/usr/local/bin/bash'),
'__bison': ('exe', 'required', '/usr/local/bin/bison'),
'__git': ('exe', 'required', '/usr/local/bin/git'),

View File

@ -35,23 +35,23 @@ def load():
processors = '/bin/grep processor /proc/cpuinfo'
e = execute.capture_execution()
exit_code, proc, output = e.shell(processors)
ncpus = 0
if exit_code == 0:
cpus = 0
try:
for l in output.split('\n'):
count = l.split(':')[1].strip()
if count > cpus:
cpus = int(count)
if cpus > 0:
smp_mflags = '-j%d' % (cpus + 1)
ncpus = int(count)
except:
pass
ncpus = str(ncpus + 1)
if uname[4].startswith('arm'):
cpu = 'arm'
else:
cpu = uname[4]
defines = {
'_ncpus': ('none', 'none', ncpus),
'_os': ('none', 'none', 'linux'),
'_host': ('triplet', 'required', cpu + '-linux-gnu'),
'_host_vendor': ('none', 'none', 'gnu'),
@ -62,7 +62,6 @@ def load():
'_usr': ('dir', 'required', '/usr'),
'_var': ('dir', 'required', '/var'),
'optflags': ('none', 'none', '-O2 -pipe'),
'_smp_mflags': ('none', 'none', smp_mflags),
'__bzip2': ('exe', 'required', '/usr/bin/bzip2'),
'__gzip': ('exe', 'required', '/bin/gzip'),
'__tar': ('exe', 'required', '/bin/tar')

View File

@ -138,7 +138,7 @@ class buildset:
self.copy(src, dst)
def canadian_cross(self, _build):
defaults_to_save = ['%{_prefix}',
defaults_to_save = ['%{_prefix}',
'%{_tmproot}',
'%{buildroot}',
'%{_builddir}',
@ -155,7 +155,7 @@ class buildset:
_build.make()
for d in defaults_to_save:
_build.config.set_define(d, orig_defaults[d])
self.root_copy(_build.config.expand('%{buildcxcroot}'),
self.root_copy(_build.config.expand('%{buildcxcroot}'),
_build.config.expand('%{_tmpcxcroot}'))
def build_package(self, _config, _build):
@ -163,7 +163,7 @@ class buildset:
self.canadian_cross(_build)
_build.make()
self.report(_config, _build)
self.root_copy(_build.config.expand('%{buildroot}'),
self.root_copy(_build.config.expand('%{buildroot}'),
_build.config.expand('%{_tmproot}'))
def bset_tar(self, _build):
@ -275,7 +275,7 @@ class buildset:
_trace(self.opts, '_bset:%s: configs: %s' % (self.bset, ','.join(configs)))
current_path = os.environ['PATH']
start = datetime.datetime.now()
try:
@ -311,17 +311,19 @@ class buildset:
else:
raise error.general('invalid config type: %s' % (configs[s]))
except error.general, gerr:
if self.opts.get_arg('--keep-going'):
if self.opts.keep_going():
print gerr
if self.opts.always_clean():
builds += [b]
else:
raise
if deps is None and not self.opts.get_arg('--no-install'):
for b in builds:
self.install(b.name(),
b.config.expand('%{buildroot}'),
b.config.expand('%{buildroot}'),
b.config.expand('%{_prefix}'))
if deps is None and \
(not self.opts.no_clean() or self.opts.get_arg('--keep-going')):
(not self.opts.no_clean() or self.opts.always_clean()):
for b in builds:
_notice(self.opts, 'cleaning: %s' % (b.name()))
b.cleanup()
@ -357,7 +359,6 @@ def run():
optargs = { '--list-configs': 'List available configurations',
'--list-bsets': 'List available build sets',
'--list-deps': 'List the dependent files.',
'--keep-going': 'Do not stop on error.',
'--no-install': 'Do not install the packages to the prefix.',
'--no-report': 'Do not create a package report.',
'--report-format': 'The report format (text, html, asciidoc).',

View File

@ -58,15 +58,12 @@ def load():
pass
if os.environ.has_key('NUMBER_OF_PROCESSORS'):
ncpus = int(os.environ['NUMBER_OF_PROCESSORS'])
ncpus = os.environ['NUMBER_OF_PROCESSORS']
else:
ncpus = 0
if ncpus > 1:
smp_mflags = '-j' + str(ncpus)
else:
smp_mflags = ''
ncpus = '1'
defines = {
'_ncpus': ('none', 'none', ncpus),
'_os': ('none', 'none', 'win32'),
'_build': ('triplet', 'required', build_triple),
'_host': ('triplet', 'required', host_triple),