rtems/rtems-source-builder
1
0
Fork 0
mirror of https://git.rtems.org/rtems-source-builder synced 2024-10-09 07:15:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

sb/get-sources: Fix getting sources with archs

Browse Source 
- Add a stop on error option

- Add listing root or toplevel buildset files

- Add used and unused file output to track what is not used
  for clean ups

- Update to handle macro expanded includes in buildset files
This commit is contained in:
Chris Johns 2022-08-14 22:04:40 +10:00
parent 4fd9fbd7c9
commit cbcf8698a8
2 changed files with 123 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
source-builder/sb/getsources.py
View File

@ -60,7 +60,9 @@ def run(args = sys.argv):
default = version.version())
argsp.add_argument('--list-hosts', help = 'List the hosts.',
action = 'store_true')
argsp.add_argument('--list-bsets', help = 'List the hosts.',
argsp.add_argument('--list-bsets', help = 'List the buildsets.',
action = 'store_true')
argsp.add_argument('--list-root-bsets', help = 'List the toplevel or root buildsets.',
action = 'store_true')
argsp.add_argument('--download-dir', help = 'Download directory.',
type = str)
@ -71,8 +73,14 @@ def run(args = sys.argv):
argsp.add_argument('--log', help = 'Log file.',
type = str,
default = simhost.log_default('getsource'))
argsp.add_argument('--stop-on-error', help = 'Stop on error.',
action = 'store_true')
argsp.add_argument('--trace', help = 'Enable trace logging for debugging.',
action = 'store_true')
argsp.add_argument('--used', help = 'Save the used buildset and config files.',
type = str, default = None)
argsp.add_argument('--unused', help = 'Save the unused buildset and config files.',
type = str, default = None)
argsp.add_argument('bsets', nargs='*', help = 'Build sets.')
argopts = argsp.parse_args(args[1:])
@ -84,10 +92,14 @@ def run(args = sys.argv):
opts = simhost.load_options(args, argopts, extras = ['--with-download'])
configs = build.get_configs(opts)
stop_on_error = argopts.stop_on_error
if argopts.list_hosts:
simhost.list_hosts()
elif argopts.list_bsets:
simhost.list_bset_files(opts, configs)
elif argopts.list_root_bsets:
simhost.list_root_bset_files(opts, configs)
else:
if argopts.clean:
if argopts.download_dir is None:
@ -95,11 +107,11 @@ def run(args = sys.argv):
if path.exists(argopts.download_dir):
log.notice('Cleaning source directory: %s' % (argopts.download_dir))
path.removeall(argopts.download_dir)
all_bsets = simhost.get_bset_files(configs)
if len(argopts.bsets) == 0:
bsets = all_bsets
bsets = simhost.get_root_bset_files(opts, configs)
else:
bsets = argopts.bsets
deps = copy.copy(simhost.strip_common_prefix(bsets))
for bset in bsets:
b = None
try:
@ -108,11 +120,23 @@ def run(args = sys.argv):
b = simhost.buildset(bset, configs, opts)
get_sources_error = False
b.build(host)
deps += b.deps()
del b
except error.general as gerr:
if stop_on_error:
raise
log.stderr(str(gerr))
log.stderr('Build FAILED')
b = None
deps = sorted(list(set(deps)))
if argopts.used:
with open(argopts.used, 'w') as o:
o.write(os.linesep.join(deps))
if argopts.unused:
cfgs_bsets = \
[cb for cb in simhost.get_config_bset_files(opts, configs) if not cb in deps]
with open(argopts.unused, 'w') as o:
o.write(os.linesep.join(cfgs_bsets))
except error.general as gerr:
if get_sources_error:
log.stderr(str(gerr))

112
source-builder/sb/simhost.py
View File

@ -36,6 +36,7 @@ try:
from . import log
from . import macros
from . import path
from . import shell
from . import sources
from . import version
except KeyboardInterrupt:
@ -122,7 +123,6 @@ profiles = {
'_var': ('dir', 'optional', '/usr/local/var') },
}
class log_capture(object):
def __init__(self):
self.log = []
@ -154,6 +154,18 @@ def find_bset_config(bset_config, macros):
raise error.general('no build set file found: %s' % (bset_config))
return name
def macro_expand(macros, _str):
cstr = None
while cstr != _str:
cstr = _str
_str = macros.expand(_str)
_str = shell.expand(macros, _str)
return _str
def strip_common_prefix(files):
commonprefix = os.path.commonprefix(files)
return sorted(list(set([f[len(commonprefix):] for f in files])))
#
# A skinny options command line class to get the configs to load.
#
@ -252,10 +264,10 @@ class options(object):
a += 1
return None
def rtems_bsp(self):
def rtems_bsp(self, arch='arch'):
self.defaults['rtems_version'] = str(version.version())
self.defaults['_target'] = 'arch-rtems'
self.defaults['rtems_host'] = 'rtems-arch'
self.defaults['_target'] = arch + '-rtems'
self.defaults['rtems_host'] = 'rtems-' + arch
self.defaults['with_rtems_bsp'] = 'rtems-bsp'
def sb_git(self):
@ -383,8 +395,19 @@ class buildset:
rebased += [i]
return rebased
def root(self):
for i in self._includes:
si = i.split(':')
if len(si) == 2:
if si[1] == 'root':
return si[0]
return None
def includes(self):
return sorted(list(set(self._includes)))
return [i for i in self._includes if not i.endswith(':root')]
def deps(self):
return strip_common_prefix([i.split(':')[0] for i in self.includes()])
def errors(self):
return sorted(list(set(self._errors)))
@ -393,7 +416,7 @@ class buildset:
if not _build.disabled():
_build.make()
def parse(self, bset):
def parse(self, bset, expand=True):
#
# Ouch, this is a copy of the setbuilder.py code.
@ -409,7 +432,7 @@ class buildset:
bsetname = find_bset_config(bset, self.macros)
try:
log.trace('_bset: %s: open: %s' % (self.bset, bsetname))
log.trace('_bset: %s: open: %s %s' % (self.bset, bsetname, expand))
bsetf = open(path.host(bsetname), 'r')
except IOError as err:
raise error.general('error opening bset file: %s' % (bsetname))
@ -432,19 +455,26 @@ class buildset:
if ls[0][-1] == ':' and ls[0][:-1] == 'package':
self.bset_pkg = ls[1].strip()
self.macros['package'] = self.bset_pkg
elif ls[0][0] == '%':
elif ls[0][0] == '%' and (len(ls[0]) > 1 and ls[0][1] != '{'):
def err(msg):
raise error.general('%s:%d: %s' % (self.bset, lc, msg))
if ls[0] == '%define':
if ls[0] == '%define' or ls[0] == '%defineifnot' :
name = ls[1].strip()
value = None
if len(ls) > 2:
self.macros.define(ls[1].strip(),
' '.join([f.strip() for f in ls[2:]]))
value = ' '.join([f.strip() for f in ls[2:]])
if ls[0] == '%defineifnot':
if self.macros.defined(name):
name = None
if name is not None:
if value is not None:
self.macros.define(name, value)
else:
self.macros.define(ls[1].strip())
self.macros.define(name)
elif ls[0] == '%undefine':
if len(ls) > 2:
raise error.general('%s:%d: %undefine requires just the name' \
% (self.bset, lc))
raise error.general('%s:%d: %undefine requires ' \
'just the name' % (self.bset, lc))
self.macros.undefine(ls[1].strip())
elif ls[0] == '%include':
configs += self.parse(ls[1].strip())
@ -453,7 +483,13 @@ class buildset:
elif ls[0] == '%hash':
sources.hash(ls[1:], self.macros, err)
else:
l = l.strip()
try:
l = macro_expand(self.macros, l.strip())
except:
if expand:
raise
l = None
if l is not None:
c = build.find_config(l, self.configs)
if c is None:
raise error.general('%s:%d: cannot find file: %s'
@ -523,7 +559,7 @@ class buildset:
nesting_count += 1
log.trace('_bset: %s for %s: make' % (self.bset, host))
log.trace('_bset: %2d: %s for %s: make' % (nesting_count, self.bset, host))
log.notice('Build Set: %s for %s' % (self.bset, host))
mail_subject = '%s on %s' % (self.bset,
@ -538,7 +574,7 @@ class buildset:
try:
configs = self.load()
log.trace('_bset: %s: configs: %s' % (self.bset, ','.join(configs)))
log.trace('_bset: %2d: %s: configs: %s' % (nesting_count, self.bset, ','.join(configs)))
sizes_valid = False
builds = []
@ -556,14 +592,14 @@ class buildset:
self.set_host_details(host, opts, macros)
config, parent = configs[s].split(':', 2)
if config.endswith('.bset'):
log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75))
log.trace('_bset: %2d: %s' % (nesting_count + 1, '=' * 75))
bs = buildset(config, self.configs, opts, macros)
bs.build(host, nesting_count)
self._includes += \
self._rebase_includes(bs.includes(), parent)
del bs
elif config.endswith('.cfg'):
log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75))
log.trace('_bset: %2d: %s' % (nesting_count + 1, '-' * 75))
try:
b = build.build(config,
False,
@ -579,7 +615,7 @@ class buildset:
#
# Dump post build macros.
#
log.trace('_bset: macros post-build')
log.trace('_bset: %2d: macros post-build' % (nesting_count))
log.trace(str(macros))
else:
raise error.general('invalid config type: %s' % (config))
@ -589,13 +625,16 @@ class buildset:
if self.build_failure is None:
self.build_failure = b.name()
self._includes += b.includes()
self._errors += [find_bset_config(config, opts.defaults) + ':' + parent] + self._includes
self._errors += \
[find_bset_config(config, opts.defaults) + ':' + parent] + self._includes
raise
#
# Clear out the builds ...
#
for b in builds:
del b
self._includes += \
[find_bset_config(c.split(':')[0], self.macros) + ':' + self.bset for c in configs]
except error.general as gerr:
if not build_error:
log.stderr(str(gerr))
@ -616,7 +655,7 @@ def list_hosts():
max_os_len = max(len(h) for h in hosts)
max_host_len = max(len(profiles[h]['_host'][2]) for h in hosts)
for h in hosts:
print('%*s: %-*s %s' % (max_os_len, h, max_host_len,
log.notice('%*s: %-*s %s' % (max_os_len, h, max_host_len,
profiles[h]['_host'][2],
profiles[h]['_host'][2]))
@ -635,14 +674,39 @@ def get_config_files(configs, localpath = False):
def get_bset_files(configs, localpath = False):
return get_files(configs, '.bset', localpath)
def get_config_bset_files(opts, configs):
cbs = get_config_files(configs) + get_bset_files(configs)
return strip_common_prefix([find_bset_config(cb, opts.defaults) for cb in cbs])
def get_root_bset_files(opts, configs, localpath = False):
bsets = get_bset_files(configs, localpath)
incs = {}
for bs in bsets:
bset = buildset(bs, configs, opts)
cfgs = [find_bset_config(c.split(':')[0], bset.macros) for c in bset.parse(bs, False)]
incs[bset.root()] = bset.includes() + cfgs
roots = sorted(incs.keys())
for inc in incs:
for i in incs[inc]:
si = i.split(':')
if len(si) > 0 and si[0] in roots:
roots.remove(si[0])
return roots
def get_root(configs):
return configs['root']
def list_root_bset_files(opts, configs):
for p in configs['paths']:
log.notice('Examining: %s' % (os.path.relpath(p)))
for r in strip_common_prefix(get_root_bset_files(opts, configs)):
log.notice(' %s' % (r))
def list_bset_files(opts, configs):
for p in configs['paths']:
print('Examining: %s' % (os.path.relpath(p)))
log.notice('Examining: %s' % (os.path.relpath(p)))
for b in get_bset_files(configs):
print(' %s' % (b[:b.rfind('.')]))
log.notice(' %s' % (b[:b.rfind('.')]))
def load_log(logfile):
log.default = log.log(streams = [logfile])