mirror of
https://git.rtems.org/rtems-source-builder
synced 2024-10-09 07:15:10 +08:00
sb/get-sources: Fix getting sources with archs
- Add a stop on error option - Add listing root or toplevel buildset files - Add used and unused file output to track what is not used for clean ups - Update to handle macro expanded includes in buildset files
This commit is contained in:
Chris Johns
parent
4fd9fbd7c9
commit
cbcf8698a8
@ -60,7 +60,9 @@ def run(args = sys.argv):
|
|||||||
default = version.version())
|
default = version.version())
|
||||||
argsp.add_argument('--list-hosts', help = 'List the hosts.',
|
argsp.add_argument('--list-hosts', help = 'List the hosts.',
|
||||||
action = 'store_true')
|
action = 'store_true')
|
||||||
argsp.add_argument('--list-bsets', help = 'List the hosts.',
|
argsp.add_argument('--list-bsets', help = 'List the buildsets.',
|
||||||
|
action = 'store_true')
|
||||||
|
argsp.add_argument('--list-root-bsets', help = 'List the toplevel or root buildsets.',
|
||||||
action = 'store_true')
|
action = 'store_true')
|
||||||
argsp.add_argument('--download-dir', help = 'Download directory.',
|
argsp.add_argument('--download-dir', help = 'Download directory.',
|
||||||
type = str)
|
type = str)
|
||||||
@ -71,8 +73,14 @@ def run(args = sys.argv):
|
|||||||
argsp.add_argument('--log', help = 'Log file.',
|
argsp.add_argument('--log', help = 'Log file.',
|
||||||
type = str,
|
type = str,
|
||||||
default = simhost.log_default('getsource'))
|
default = simhost.log_default('getsource'))
|
||||||
|
argsp.add_argument('--stop-on-error', help = 'Stop on error.',
|
||||||
|
action = 'store_true')
|
||||||
argsp.add_argument('--trace', help = 'Enable trace logging for debugging.',
|
argsp.add_argument('--trace', help = 'Enable trace logging for debugging.',
|
||||||
action = 'store_true')
|
action = 'store_true')
|
||||||
|
argsp.add_argument('--used', help = 'Save the used buildset and config files.',
|
||||||
|
type = str, default = None)
|
||||||
|
argsp.add_argument('--unused', help = 'Save the unused buildset and config files.',
|
||||||
|
type = str, default = None)
|
||||||
argsp.add_argument('bsets', nargs='*', help = 'Build sets.')
|
argsp.add_argument('bsets', nargs='*', help = 'Build sets.')
|
||||||
|
|
||||||
argopts = argsp.parse_args(args[1:])
|
argopts = argsp.parse_args(args[1:])
|
||||||
@ -84,10 +92,14 @@ def run(args = sys.argv):
|
|||||||
opts = simhost.load_options(args, argopts, extras = ['--with-download'])
|
opts = simhost.load_options(args, argopts, extras = ['--with-download'])
|
||||||
configs = build.get_configs(opts)
|
configs = build.get_configs(opts)
|
||||||
|
|
||||||
|
stop_on_error = argopts.stop_on_error
|
||||||
|
|
||||||
if argopts.list_hosts:
|
if argopts.list_hosts:
|
||||||
simhost.list_hosts()
|
simhost.list_hosts()
|
||||||
elif argopts.list_bsets:
|
elif argopts.list_bsets:
|
||||||
simhost.list_bset_files(opts, configs)
|
simhost.list_bset_files(opts, configs)
|
||||||
|
elif argopts.list_root_bsets:
|
||||||
|
simhost.list_root_bset_files(opts, configs)
|
||||||
else:
|
else:
|
||||||
if argopts.clean:
|
if argopts.clean:
|
||||||
if argopts.download_dir is None:
|
if argopts.download_dir is None:
|
||||||
@ -95,11 +107,11 @@ def run(args = sys.argv):
|
|||||||
if path.exists(argopts.download_dir):
|
if path.exists(argopts.download_dir):
|
||||||
log.notice('Cleaning source directory: %s' % (argopts.download_dir))
|
log.notice('Cleaning source directory: %s' % (argopts.download_dir))
|
||||||
path.removeall(argopts.download_dir)
|
path.removeall(argopts.download_dir)
|
||||||
all_bsets = simhost.get_bset_files(configs)
|
|
||||||
if len(argopts.bsets) == 0:
|
if len(argopts.bsets) == 0:
|
||||||
bsets = all_bsets
|
bsets = simhost.get_root_bset_files(opts, configs)
|
||||||
else:
|
else:
|
||||||
bsets = argopts.bsets
|
bsets = argopts.bsets
|
||||||
|
deps = copy.copy(simhost.strip_common_prefix(bsets))
|
||||||
for bset in bsets:
|
for bset in bsets:
|
||||||
b = None
|
b = None
|
||||||
try:
|
try:
|
||||||
@ -108,11 +120,23 @@ def run(args = sys.argv):
|
|||||||
b = simhost.buildset(bset, configs, opts)
|
b = simhost.buildset(bset, configs, opts)
|
||||||
get_sources_error = False
|
get_sources_error = False
|
||||||
b.build(host)
|
b.build(host)
|
||||||
|
deps += b.deps()
|
||||||
del b
|
del b
|
||||||
except error.general as gerr:
|
except error.general as gerr:
|
||||||
|
if stop_on_error:
|
||||||
|
raise
|
||||||
log.stderr(str(gerr))
|
log.stderr(str(gerr))
|
||||||
log.stderr('Build FAILED')
|
log.stderr('Build FAILED')
|
||||||
b = None
|
b = None
|
||||||
|
deps = sorted(list(set(deps)))
|
||||||
|
if argopts.used:
|
||||||
|
with open(argopts.used, 'w') as o:
|
||||||
|
o.write(os.linesep.join(deps))
|
||||||
|
if argopts.unused:
|
||||||
|
cfgs_bsets = \
|
||||||
|
[cb for cb in simhost.get_config_bset_files(opts, configs) if not cb in deps]
|
||||||
|
with open(argopts.unused, 'w') as o:
|
||||||
|
o.write(os.linesep.join(cfgs_bsets))
|
||||||
except error.general as gerr:
|
except error.general as gerr:
|
||||||
if get_sources_error:
|
if get_sources_error:
|
||||||
log.stderr(str(gerr))
|
log.stderr(str(gerr))
|
||||||
|
|||||||
@ -36,6 +36,7 @@ try:
|
|||||||
from . import log
|
from . import log
|
||||||
from . import macros
|
from . import macros
|
||||||
from . import path
|
from . import path
|
||||||
|
from . import shell
|
||||||
from . import sources
|
from . import sources
|
||||||
from . import version
|
from . import version
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
@ -122,7 +123,6 @@ profiles = {
|
|||||||
'_var': ('dir', 'optional', '/usr/local/var') },
|
'_var': ('dir', 'optional', '/usr/local/var') },
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class log_capture(object):
|
class log_capture(object):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.log = []
|
self.log = []
|
||||||
@ -154,6 +154,18 @@ def find_bset_config(bset_config, macros):
|
|||||||
raise error.general('no build set file found: %s' % (bset_config))
|
raise error.general('no build set file found: %s' % (bset_config))
|
||||||
return name
|
return name
|
||||||
|
|
||||||
|
def macro_expand(macros, _str):
|
||||||
|
cstr = None
|
||||||
|
while cstr != _str:
|
||||||
|
cstr = _str
|
||||||
|
_str = macros.expand(_str)
|
||||||
|
_str = shell.expand(macros, _str)
|
||||||
|
return _str
|
||||||
|
|
||||||
|
def strip_common_prefix(files):
|
||||||
|
commonprefix = os.path.commonprefix(files)
|
||||||
|
return sorted(list(set([f[len(commonprefix):] for f in files])))
|
||||||
|
|
||||||
#
|
#
|
||||||
# A skinny options command line class to get the configs to load.
|
# A skinny options command line class to get the configs to load.
|
||||||
#
|
#
|
||||||
@ -252,10 +264,10 @@ class options(object):
|
|||||||
a += 1
|
a += 1
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def rtems_bsp(self):
|
def rtems_bsp(self, arch='arch'):
|
||||||
self.defaults['rtems_version'] = str(version.version())
|
self.defaults['rtems_version'] = str(version.version())
|
||||||
self.defaults['_target'] = 'arch-rtems'
|
self.defaults['_target'] = arch + '-rtems'
|
||||||
self.defaults['rtems_host'] = 'rtems-arch'
|
self.defaults['rtems_host'] = 'rtems-' + arch
|
||||||
self.defaults['with_rtems_bsp'] = 'rtems-bsp'
|
self.defaults['with_rtems_bsp'] = 'rtems-bsp'
|
||||||
|
|
||||||
def sb_git(self):
|
def sb_git(self):
|
||||||
@ -383,8 +395,19 @@ class buildset:
|
|||||||
rebased += [i]
|
rebased += [i]
|
||||||
return rebased
|
return rebased
|
||||||
|
|
||||||
|
def root(self):
|
||||||
|
for i in self._includes:
|
||||||
|
si = i.split(':')
|
||||||
|
if len(si) == 2:
|
||||||
|
if si[1] == 'root':
|
||||||
|
return si[0]
|
||||||
|
return None
|
||||||
|
|
||||||
def includes(self):
|
def includes(self):
|
||||||
return sorted(list(set(self._includes)))
|
return [i for i in self._includes if not i.endswith(':root')]
|
||||||
|
|
||||||
|
def deps(self):
|
||||||
|
return strip_common_prefix([i.split(':')[0] for i in self.includes()])
|
||||||
|
|
||||||
def errors(self):
|
def errors(self):
|
||||||
return sorted(list(set(self._errors)))
|
return sorted(list(set(self._errors)))
|
||||||
@ -393,7 +416,7 @@ class buildset:
|
|||||||
if not _build.disabled():
|
if not _build.disabled():
|
||||||
_build.make()
|
_build.make()
|
||||||
|
|
||||||
def parse(self, bset):
|
def parse(self, bset, expand=True):
|
||||||
|
|
||||||
#
|
#
|
||||||
# Ouch, this is a copy of the setbuilder.py code.
|
# Ouch, this is a copy of the setbuilder.py code.
|
||||||
@ -409,7 +432,7 @@ class buildset:
|
|||||||
bsetname = find_bset_config(bset, self.macros)
|
bsetname = find_bset_config(bset, self.macros)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
log.trace('_bset: %s: open: %s' % (self.bset, bsetname))
|
log.trace('_bset: %s: open: %s %s' % (self.bset, bsetname, expand))
|
||||||
bsetf = open(path.host(bsetname), 'r')
|
bsetf = open(path.host(bsetname), 'r')
|
||||||
except IOError as err:
|
except IOError as err:
|
||||||
raise error.general('error opening bset file: %s' % (bsetname))
|
raise error.general('error opening bset file: %s' % (bsetname))
|
||||||
@ -432,19 +455,26 @@ class buildset:
|
|||||||
if ls[0][-1] == ':' and ls[0][:-1] == 'package':
|
if ls[0][-1] == ':' and ls[0][:-1] == 'package':
|
||||||
self.bset_pkg = ls[1].strip()
|
self.bset_pkg = ls[1].strip()
|
||||||
self.macros['package'] = self.bset_pkg
|
self.macros['package'] = self.bset_pkg
|
||||||
elif ls[0][0] == '%':
|
elif ls[0][0] == '%' and (len(ls[0]) > 1 and ls[0][1] != '{'):
|
||||||
def err(msg):
|
def err(msg):
|
||||||
raise error.general('%s:%d: %s' % (self.bset, lc, msg))
|
raise error.general('%s:%d: %s' % (self.bset, lc, msg))
|
||||||
if ls[0] == '%define':
|
if ls[0] == '%define' or ls[0] == '%defineifnot' :
|
||||||
|
name = ls[1].strip()
|
||||||
|
value = None
|
||||||
if len(ls) > 2:
|
if len(ls) > 2:
|
||||||
self.macros.define(ls[1].strip(),
|
value = ' '.join([f.strip() for f in ls[2:]])
|
||||||
' '.join([f.strip() for f in ls[2:]]))
|
if ls[0] == '%defineifnot':
|
||||||
else:
|
if self.macros.defined(name):
|
||||||
self.macros.define(ls[1].strip())
|
name = None
|
||||||
|
if name is not None:
|
||||||
|
if value is not None:
|
||||||
|
self.macros.define(name, value)
|
||||||
|
else:
|
||||||
|
self.macros.define(name)
|
||||||
elif ls[0] == '%undefine':
|
elif ls[0] == '%undefine':
|
||||||
if len(ls) > 2:
|
if len(ls) > 2:
|
||||||
raise error.general('%s:%d: %undefine requires just the name' \
|
raise error.general('%s:%d: %undefine requires ' \
|
||||||
% (self.bset, lc))
|
'just the name' % (self.bset, lc))
|
||||||
self.macros.undefine(ls[1].strip())
|
self.macros.undefine(ls[1].strip())
|
||||||
elif ls[0] == '%include':
|
elif ls[0] == '%include':
|
||||||
configs += self.parse(ls[1].strip())
|
configs += self.parse(ls[1].strip())
|
||||||
@ -453,12 +483,18 @@ class buildset:
|
|||||||
elif ls[0] == '%hash':
|
elif ls[0] == '%hash':
|
||||||
sources.hash(ls[1:], self.macros, err)
|
sources.hash(ls[1:], self.macros, err)
|
||||||
else:
|
else:
|
||||||
l = l.strip()
|
try:
|
||||||
c = build.find_config(l, self.configs)
|
l = macro_expand(self.macros, l.strip())
|
||||||
if c is None:
|
except:
|
||||||
raise error.general('%s:%d: cannot find file: %s'
|
if expand:
|
||||||
% (self.bset, lc, l))
|
raise
|
||||||
configs += [c + ':' + self.parent]
|
l = None
|
||||||
|
if l is not None:
|
||||||
|
c = build.find_config(l, self.configs)
|
||||||
|
if c is None:
|
||||||
|
raise error.general('%s:%d: cannot find file: %s'
|
||||||
|
% (self.bset, lc, l))
|
||||||
|
configs += [c + ':' + self.parent]
|
||||||
finally:
|
finally:
|
||||||
bsetf.close()
|
bsetf.close()
|
||||||
self.parent = parent
|
self.parent = parent
|
||||||
@ -523,7 +559,7 @@ class buildset:
|
|||||||
|
|
||||||
nesting_count += 1
|
nesting_count += 1
|
||||||
|
|
||||||
log.trace('_bset: %s for %s: make' % (self.bset, host))
|
log.trace('_bset: %2d: %s for %s: make' % (nesting_count, self.bset, host))
|
||||||
log.notice('Build Set: %s for %s' % (self.bset, host))
|
log.notice('Build Set: %s for %s' % (self.bset, host))
|
||||||
|
|
||||||
mail_subject = '%s on %s' % (self.bset,
|
mail_subject = '%s on %s' % (self.bset,
|
||||||
@ -538,7 +574,7 @@ class buildset:
|
|||||||
try:
|
try:
|
||||||
configs = self.load()
|
configs = self.load()
|
||||||
|
|
||||||
log.trace('_bset: %s: configs: %s' % (self.bset, ','.join(configs)))
|
log.trace('_bset: %2d: %s: configs: %s' % (nesting_count, self.bset, ','.join(configs)))
|
||||||
|
|
||||||
sizes_valid = False
|
sizes_valid = False
|
||||||
builds = []
|
builds = []
|
||||||
@ -556,14 +592,14 @@ class buildset:
|
|||||||
self.set_host_details(host, opts, macros)
|
self.set_host_details(host, opts, macros)
|
||||||
config, parent = configs[s].split(':', 2)
|
config, parent = configs[s].split(':', 2)
|
||||||
if config.endswith('.bset'):
|
if config.endswith('.bset'):
|
||||||
log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75))
|
log.trace('_bset: %2d: %s' % (nesting_count + 1, '=' * 75))
|
||||||
bs = buildset(config, self.configs, opts, macros)
|
bs = buildset(config, self.configs, opts, macros)
|
||||||
bs.build(host, nesting_count)
|
bs.build(host, nesting_count)
|
||||||
self._includes += \
|
self._includes += \
|
||||||
self._rebase_includes(bs.includes(), parent)
|
self._rebase_includes(bs.includes(), parent)
|
||||||
del bs
|
del bs
|
||||||
elif config.endswith('.cfg'):
|
elif config.endswith('.cfg'):
|
||||||
log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75))
|
log.trace('_bset: %2d: %s' % (nesting_count + 1, '-' * 75))
|
||||||
try:
|
try:
|
||||||
b = build.build(config,
|
b = build.build(config,
|
||||||
False,
|
False,
|
||||||
@ -579,7 +615,7 @@ class buildset:
|
|||||||
#
|
#
|
||||||
# Dump post build macros.
|
# Dump post build macros.
|
||||||
#
|
#
|
||||||
log.trace('_bset: macros post-build')
|
log.trace('_bset: %2d: macros post-build' % (nesting_count))
|
||||||
log.trace(str(macros))
|
log.trace(str(macros))
|
||||||
else:
|
else:
|
||||||
raise error.general('invalid config type: %s' % (config))
|
raise error.general('invalid config type: %s' % (config))
|
||||||
@ -589,13 +625,16 @@ class buildset:
|
|||||||
if self.build_failure is None:
|
if self.build_failure is None:
|
||||||
self.build_failure = b.name()
|
self.build_failure = b.name()
|
||||||
self._includes += b.includes()
|
self._includes += b.includes()
|
||||||
self._errors += [find_bset_config(config, opts.defaults) + ':' + parent] + self._includes
|
self._errors += \
|
||||||
|
[find_bset_config(config, opts.defaults) + ':' + parent] + self._includes
|
||||||
raise
|
raise
|
||||||
#
|
#
|
||||||
# Clear out the builds ...
|
# Clear out the builds ...
|
||||||
#
|
#
|
||||||
for b in builds:
|
for b in builds:
|
||||||
del b
|
del b
|
||||||
|
self._includes += \
|
||||||
|
[find_bset_config(c.split(':')[0], self.macros) + ':' + self.bset for c in configs]
|
||||||
except error.general as gerr:
|
except error.general as gerr:
|
||||||
if not build_error:
|
if not build_error:
|
||||||
log.stderr(str(gerr))
|
log.stderr(str(gerr))
|
||||||
@ -616,9 +655,9 @@ def list_hosts():
|
|||||||
max_os_len = max(len(h) for h in hosts)
|
max_os_len = max(len(h) for h in hosts)
|
||||||
max_host_len = max(len(profiles[h]['_host'][2]) for h in hosts)
|
max_host_len = max(len(profiles[h]['_host'][2]) for h in hosts)
|
||||||
for h in hosts:
|
for h in hosts:
|
||||||
print('%*s: %-*s %s' % (max_os_len, h, max_host_len,
|
log.notice('%*s: %-*s %s' % (max_os_len, h, max_host_len,
|
||||||
profiles[h]['_host'][2],
|
profiles[h]['_host'][2],
|
||||||
profiles[h]['_host'][2]))
|
profiles[h]['_host'][2]))
|
||||||
|
|
||||||
def get_files(configs, ext, localpath):
|
def get_files(configs, ext, localpath):
|
||||||
files = []
|
files = []
|
||||||
@ -635,14 +674,39 @@ def get_config_files(configs, localpath = False):
|
|||||||
def get_bset_files(configs, localpath = False):
|
def get_bset_files(configs, localpath = False):
|
||||||
return get_files(configs, '.bset', localpath)
|
return get_files(configs, '.bset', localpath)
|
||||||
|
|
||||||
|
def get_config_bset_files(opts, configs):
|
||||||
|
cbs = get_config_files(configs) + get_bset_files(configs)
|
||||||
|
return strip_common_prefix([find_bset_config(cb, opts.defaults) for cb in cbs])
|
||||||
|
|
||||||
|
def get_root_bset_files(opts, configs, localpath = False):
|
||||||
|
bsets = get_bset_files(configs, localpath)
|
||||||
|
incs = {}
|
||||||
|
for bs in bsets:
|
||||||
|
bset = buildset(bs, configs, opts)
|
||||||
|
cfgs = [find_bset_config(c.split(':')[0], bset.macros) for c in bset.parse(bs, False)]
|
||||||
|
incs[bset.root()] = bset.includes() + cfgs
|
||||||
|
roots = sorted(incs.keys())
|
||||||
|
for inc in incs:
|
||||||
|
for i in incs[inc]:
|
||||||
|
si = i.split(':')
|
||||||
|
if len(si) > 0 and si[0] in roots:
|
||||||
|
roots.remove(si[0])
|
||||||
|
return roots
|
||||||
|
|
||||||
def get_root(configs):
|
def get_root(configs):
|
||||||
return configs['root']
|
return configs['root']
|
||||||
|
|
||||||
|
def list_root_bset_files(opts, configs):
|
||||||
|
for p in configs['paths']:
|
||||||
|
log.notice('Examining: %s' % (os.path.relpath(p)))
|
||||||
|
for r in strip_common_prefix(get_root_bset_files(opts, configs)):
|
||||||
|
log.notice(' %s' % (r))
|
||||||
|
|
||||||
def list_bset_files(opts, configs):
|
def list_bset_files(opts, configs):
|
||||||
for p in configs['paths']:
|
for p in configs['paths']:
|
||||||
print('Examining: %s' % (os.path.relpath(p)))
|
log.notice('Examining: %s' % (os.path.relpath(p)))
|
||||||
for b in get_bset_files(configs):
|
for b in get_bset_files(configs):
|
||||||
print(' %s' % (b[:b.rfind('.')]))
|
log.notice(' %s' % (b[:b.rfind('.')]))
|
||||||
|
|
||||||
def load_log(logfile):
|
def load_log(logfile):
|
||||||
log.default = log.log(streams = [logfile])
|
log.default = log.log(streams = [logfile])
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user