sb/setbuilder: Support staging of build set build before installing.

- Staging lets build sets use previously built build sets.

- Fix the rtems-bsp builds to support staging.
This commit is contained in:
Chris Johns 2019-07-14 14:03:06 +10:00
parent f68f0d94c0
commit b1c2cb4f72
6 changed files with 232 additions and 54 deletions

View File

@ -3,6 +3,6 @@
#
#
# Build the kernel.
# Build the BSD library for RTEMS.
#
tools/rtems-libbsd-5

View File

@ -24,7 +24,7 @@
# If a dry-run and with download ignore errors and correct setting for tools
# and BSPs. Only after the source to download.
#
%if %{_dry_run} && %{defined with_download}
%if %{_dry_run}
%log BSP configuration errors ignored
%define rtems_bsp_error 0
%else
@ -36,7 +36,7 @@
# version. It can be defined in rtems_host.
#
%if %{defined rtems_host}
%define _host %{rtems_host}
%{triplet _host %{rtems_host}}
%endif
%if %{rtems_bsp_error}
%if %{_host} == %{nil} || %{_host} == %{_build}
@ -55,18 +55,29 @@
%endif
#
# If no tools or RTEMS provided use the prefix.
# If no tools or RTEMS provided use the prefix. If staging use the staging
# root.
#
%ifn %{defined with_tools}
%define with_tools %{_prefix}
%define rtems_waf_tools %{nil}
%if %{install_mode} == staging
%define with_tools %{stagingroot}
%define rtems_waf_tools --rtems-tools=%{with_tools}
%else
%define with_tools %{_prefix}
%define rtems_waf_tools %{nil}
%endif
%else
%define rtems_waf_tools --rtems-tools=%{with_tools}
%endif
%ifn %{defined with_rtems}
%define with_rtems %{_prefix}
%define rtems_waf_rtems %{nil}
%if %{install_mode} == staging
%define with_rtems %{stagingroot}
%define rtems_waf_rtems --rtems=%{with_rtems}
%else
%define with_rtems %{_prefix}
%define rtems_waf_rtems %{nil}
%endif
%else
%define rtems_waf_rtems --rtems=%{with_rtems}
%endif
@ -111,6 +122,7 @@
# RTEMS BSP paths
#
%define rtems_bsp_cc %{_host}-gcc
%define rtems_bsp_cxx %{_host}-g++
%define rtems_bsp_incpath %{with_rtems}/%{_host}/%{rtems_bsp}/include
%define rtems_bsp_libpath %{with_rtems}/%{_host}/%{rtems_bsp}/lib
@ -160,6 +172,8 @@
#
# Map to names used for cross compiling.
#
%define host_cc %{rtems_bsp_cc}
%define host_cxx %{rtems_bsp_cxx}
%define host_includes -I%{_includedir}
%define host_cflags %{rtems_bsp_cflags}
%define host_cxxflags %{rtems_bsp_ccflags}

View File

@ -62,7 +62,7 @@ URL: https://git.rtems.org/rtems-libbsd.git/
./waf distclean configure \
--prefix=%{_prefix} \
%{rtems_waf_tools} \
%{rtems_waf_rtems}
%{rtems_waf_rtems} \
--rtems-bsp=%{rtems_bsp_arch_bsp}
./waf build

View File

@ -75,6 +75,19 @@ build_ldflags: none, convert, ''
build_includes: none, convert, ''
build_libs: none, convert, ''
#
# Build and staging paths.
#
buildroot: dir, none, '%{_tmppath}/%{buildname}-%{_uid}'
buildcxcroot: dir, none, '%{_tmppath}/%{buildname}-%{_uid}-cxc'
buildxcroot: dir, none, '%{_tmppath}/%{buildname}-%{_uid}-xx'
stagingroot: dir, none, '%{_tmppath}/sb-%{_uid}-staging'
#
# Install mode can be installing or staging. Defaults to installing.
#
install_mode: none, none, 'installing'
# Extra path a platform can override.
_extra_path: none, none, '%{_sbdir}'
_ld_library_path: none, none, 'LD_LIBRARY_PATH'
@ -96,9 +109,6 @@ _docdir: dir, none, '%{_defaultdocdir}'
_tmppath: dir, none, '%{_topdir}/build/tmp'
_tmproot: dir, none, '%{_tmppath}/sb-%{_uid}/%{_bset_tmp}'
_tmpcxcroot: dir, none, '%{_tmppath}/sb-%{_uid}-cxc/%{_bset_tmp}'
buildroot: dir, none, '%{_tmppath}/%{buildname}-%{_uid}'
buildcxcroot: dir, none, '%{_tmppath}/%{buildname}-%{_uid}-cxc'
buildxcroot: dir, none, '%{_tmppath}/%{buildname}-%{_uid}-xx'
_datadir: dir, none, '%{_prefix}/share'
_defaultdocdir: dir, none, '%{_prefix}/share/doc'
_exeext: none, none, ''

View File

@ -41,6 +41,7 @@ try:
import options
import path
import reports
import shell
import sources
import version
except KeyboardInterrupt:
@ -50,6 +51,14 @@ except:
print('error: unknown application load error', file = sys.stderr)
sys.exit(1)
def macro_expand(macros, _str):
cstr = None
while cstr != _str:
cstr = _str
_str = macros.expand(_str)
_str = shell.expand(macros, _str)
return _str
class log_capture(object):
def __init__(self):
self.log = []
@ -71,21 +80,21 @@ class buildset:
"""Build a set builds a set of packages."""
def __init__(self, bset, _configs, opts, macros = None):
log.trace('_bset: %s: init' % (bset))
log.trace('_bset: : %s: init' % (bset))
self.configs = _configs
self.opts = opts
if macros is None:
self.macros = copy.copy(opts.defaults)
else:
self.macros = copy.copy(macros)
log.trace('_bset: %s: macro defaults' % (bset))
log.trace('_bset: : %s: macro defaults' % (bset))
log.trace(str(self.macros))
self.bset = bset
_target = self.macros.expand('%{_target}')
_target = macro_expand(self.macros, '%{_target}')
if len(_target):
pkg_prefix = _target
else:
pkg_prefix = self.macros.expand('%{_host}')
pkg_prefix = macro_expand(self.macros, '%{_host}')
self.bset_pkg = '%s-%s-set' % (pkg_prefix, self.bset)
self.mail_header = ''
self.mail_report = ''
@ -152,7 +161,7 @@ class buildset:
else:
raise error.general('invalid report format: %s' % (format))
buildroot = _build.config.abspath('%{buildroot}')
prefix = _build.macros.expand('%{_prefix}')
prefix = macro_expand(_build.macros, '%{_prefix}')
name = _build.main_package().name() + ext
log.notice('reporting: %s -> %s' % (_config, name))
if not _build.opts.get_arg('--no-report'):
@ -182,17 +191,24 @@ class buildset:
def root_copy(self, src, dst):
what = '%s -> %s' % \
(os.path.relpath(path.host(src)), os.path.relpath(path.host(dst)))
log.trace('_bset: %s: collecting: %s' % (self.bset, what))
log.trace('_bset: : %s: collecting: %s' % (self.bset, what))
self.copy(src, dst)
def install(self, name, buildroot, prefix):
dst = prefix
src = path.join(buildroot, prefix)
log.notice('installing: %s -> %s' % (name, path.host(dst)))
def install(self, mode, name, src, dst):
log.notice('%s: %s -> %s' % (mode, name, path.host(dst)))
self.copy(src, dst)
def install_mode(self):
return macro_expand(self.macros, '%{install_mode}')
def installing(self):
return self.install_mode() == 'installing'
def staging(self):
return not self.installing()
def canadian_cross(self, _build):
log.trace('_bset: Cxc for build machine: _build => _host')
log.trace('_bset: : Cxc for build machine: _build => _host')
macros_to_copy = [('%{_host}', '%{_build}'),
('%{_host_alias}', '%{_build_alias}'),
('%{_host_arch}', '%{_build_arch}'),
@ -204,7 +220,7 @@ class buildset:
('%{_builddir}', '%{_buildcxcdir}')]
cxc_macros = _build.copy_init_macros()
for m in macros_to_copy:
log.trace('_bset: Cxc: %s <= %s' % (m[0], cxc_macros[m[1]]))
log.trace('_bset: : Cxc: %s <= %s' % (m[0], cxc_macros[m[1]]))
cxc_macros[m[0]] = cxc_macros[m[1]]
_build.set_macros(cxc_macros)
_build.reload()
@ -229,12 +245,15 @@ class buildset:
if (self.opts.get_arg('--bset-tar-file') or self.opts.canadian_cross()) \
and not _build.macros.get('%{_disable_packaging}'):
path.mkdir(tardir)
tar = path.join(tardir, _build.config.expand('%s.tar.bz2' % (_build.main_package().name())))
tar = path.join(tardir,
_build.config.expand('%s.tar.bz2' % \
(_build.main_package().name())))
log.notice('tarball: %s' % (os.path.relpath(path.host(tar))))
if not self.opts.dry_run():
tmproot = _build.config.expand('%{_tmproot}')
cmd = _build.config.expand('"cd ' + tmproot + \
' && %{__tar} -cf - . | %{__bzip2} > ' + tar + '"')
' && %{__tar} -cf - . | %{__bzip2} > ' + \
tar + '"')
_build.run(cmd, shell_opts = '-c', cwd = tmproot)
def parse(self, bset):
@ -249,7 +268,7 @@ class buildset:
bsetname = bset
if not path.exists(bsetname):
for cp in self.macros.expand('%{_configdir}').split(':'):
for cp in macro_expand(self.macros, '%{_configdir}').split(':'):
configdir = path.abspath(cp)
bsetname = path.join(configdir, bset)
if path.exists(bsetname):
@ -258,7 +277,7 @@ class buildset:
if bsetname is None:
raise error.general('no build set file found: %s' % (bset))
try:
log.trace('_bset: %s: open: %s' % (self.bset, bsetname))
log.trace('_bset: : %s: open: %s' % (self.bset, bsetname))
bset = open(path.host(bsetname), 'r')
except IOError as err:
raise error.general('error opening bset file: %s' % (bsetname))
@ -272,7 +291,7 @@ class buildset:
l = _clean(l)
if len(l) == 0:
continue
log.trace('_bset: %s: %03d: %s' % (self.bset, lc, l))
log.trace('_bset: : %s: %03d: %s' % (self.bset, lc, l))
ls = l.split()
if ls[0][-1] == ':' and ls[0][:-1] == 'package':
self.bset_pkg = ls[1].strip()
@ -288,8 +307,8 @@ class buildset:
self.macros.define(ls[1].strip())
elif ls[0] == '%undefine':
if len(ls) > 2:
raise error.general('%s:%d: %undefine requires just the name' % \
(self.bset, lc))
raise error.general('%s:%d: %undefine requires ' \
'just the name' % (self.bset, lc))
self.macros.undefine(ls[1].strip())
elif ls[0] == '%include':
configs += self.parse(ls[1].strip())
@ -301,7 +320,8 @@ class buildset:
l = l.strip()
c = build.find_config(l, self.configs)
if c is None:
raise error.general('%s:%d: cannot find file: %s' % (self.bset, lc, l))
raise error.general('%s:%d: cannot find file: %s' % (self.bset,
lc, l))
configs += [c]
except:
bset.close()
@ -320,9 +340,12 @@ class buildset:
if self.bset.endswith('.cfg'):
configs = [self.bset]
else:
exbset = self.macros.expand(self.bset)
exbset = macro_expand(self.macros, self.bset)
self.macros['_bset'] = exbset
self.macros['_bset_tmp'] = build.short_name(exbset)
bset_tmp = build.short_name(exbset)
if bset_tmp.endswith('.bset'):
bset_tmp = bset_tmp[:-5]
self.macros['_bset_tmp'] = bset_tmp
root, ext = path.splitext(exbset)
if exbset.endswith('.bset'):
bset = exbset
@ -340,7 +363,7 @@ class buildset:
if mail:
mail['output'].clear()
log.trace('_bset: %s: make' % (self.bset))
log.trace('_bset: %2d: %s: make' % (nesting_count, self.bset))
log.notice('Build Set: %s' % (self.bset))
mail_subject = '%s on %s' % (self.bset,
@ -350,16 +373,44 @@ class buildset:
start = datetime.datetime.now()
mail_report = False
have_errors = False
mail_report = False
have_errors = False
if mail:
mail['output'].clear()
#
# If this is the outter most buildset it's files installed. Nested
# build sets staged their installed file. The staged files are install
# when the outtter most build finishes.
#
if nesting_count != 1:
if self.installing():
self.macros['install_mode'] = 'staging'
#
# Prepend staging areas, bin directory tothe
# path. Lets the later package depend on the eailier
# ones.
#
pathprepend = ['%{stagingroot}/bin'] + \
macro_expand(self.macros, '%{_pathprepend}').split(':')
pathprepend = [pp for pp in pathprepend if len(pp)]
if len(pathprepend) == 1:
self.macros['_pathprepend'] = pathprepend[0]
else:
self.macros['_pathprepend'] = ':'.join(pathprepend)
#
# Only the outter build set can have staging to install. Get the staging
# root via the config because it could require a valid config.
#
have_staging = False
try:
configs = self.load()
log.trace('_bset: %s: configs: %s' % (self.bset, ','.join(configs)))
log.trace('_bset: %2d: %s: configs: %s' % (nesting_count,
self.bset, ', '.join(configs)))
sizes_valid = False
builds = []
@ -374,14 +425,20 @@ class buildset:
opts = copy.copy(self.opts)
macros = copy.copy(self.macros)
if configs[s].endswith('.bset'):
log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75))
log.trace('_bset: %2d: %s %s' % (nesting_count,
configs[s],
'=' * (74 - len(configs[s]))))
bs = buildset(configs[s], self.configs, opts, macros)
bs.build(deps, nesting_count, mail)
if self.installing():
have_staging = True
del bs
elif configs[s].endswith('.cfg'):
if mail:
mail_report = True
log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75))
log.trace('_bset: %2d: %s %s' % (nesting_count,
configs[s],
'=' * (74 - len(configs[s]))))
try:
b = build.build(configs[s],
self.opts.get_arg('--pkg-tar-files'),
@ -412,7 +469,7 @@ class buildset:
#
# Dump post build macros.
#
log.trace('_bset: macros post-build')
log.trace('_bset: : macros post-build')
log.trace(str(b.macros))
else:
raise error.general('invalid config type: %s' % (configs[s]))
@ -436,19 +493,25 @@ class buildset:
else:
raise
#
# Installing ...
# Installing or staging ...
#
log.trace('_bset: installing: deps:%r no-install:%r' % \
(deps is None, self.opts.no_install()))
if deps is None \
and not self.opts.no_install() \
and not have_errors:
log.trace('_bset: %2d: %s: deps:%r no-install:%r' % \
(nesting_count, self.install_mode(),
deps is None, self.opts.no_install()))
log.trace('_bset: %2d: %s: builds: %s' % \
(nesting_count, self.install_mode(),
', '.join([b.name() for b in builds])))
if deps is None and not have_errors:
for b in builds:
log.trace('_bset: installing: %r' % b.installable())
log.trace('_bset: : %s: %r' % (self.install_mode(),
b.installable()))
if b.installable():
self.install(b.name(),
b.config.expand('%{buildroot}'),
b.config.expand('%{_prefix}'))
prefix = b.config.expand('%{_prefix}')
buildroot = path.join(b.config.expand('%{buildroot}'), prefix)
if self.staging():
prefix = b.config.expand('%{stagingroot}')
self.install(self.install_mode(), b.name(), buildroot, prefix)
#
# Sizes ...
#
@ -469,7 +532,8 @@ class buildset:
for p in builds[0].config.expand('%{_patchdir}').split(':'):
size_patches += path.get_size(p)
size_total = size_sources + size_patches + size_installed
build_max_size_human = build.humanize_number(size_build_max + size_installed, 'B')
build_max_size_human = build.humanize_number(size_build_max +
size_installed, 'B')
build_total_size_human = build.humanize_number(size_total, 'B')
build_sources_size_human = build.humanize_number(size_sources, 'B')
build_patches_size_human = build.humanize_number(size_patches, 'B')
@ -499,6 +563,21 @@ class buildset:
#
for b in builds:
del b
#
# If builds have been staged install into the finaly prefix.
#
if have_staging and not self.opts.no_install() and not have_errors:
log.trace('_bset: %2d: install staging' % (nesting_count))
stagingroot = macro_expand(self.macros, '%{stagingroot}')
prefix = macro_expand(self.macros, '%{_prefix}')
self.install(self.install_mode(), self.bset, stagingroot, prefix)
staging_size = path.get_size(stagingroot)
if not self.opts.no_clean() or self.opts.always_clean():
log.notice('clean staging: %s' % (self.bset))
log.trace('cleanup: %s' % (stagingroot))
self.rmdir(stagingroot)
log.notice('Staging Size: %s' % (build.humanize_number(staging_size)))
except error.general as gerr:
if not build_error:
log.stderr(str(gerr))
@ -586,7 +665,7 @@ def run():
if to_addr is not None:
mail['to'] = to_addr[1]
else:
mail['to'] = opts.defaults.expand('%{_mail_tools_to}')
mail['to'] = macro_expand(opts.defaults, '%{_mail_tools_to}')
mail['from'] = mail['mail'].from_address()
log.notice('RTEMS Source Builder - Set Builder, %s' % (version.str()))
opts.log_info()
@ -606,7 +685,7 @@ def run():
else:
deps = None
if not list_bset_cfg_files(opts, configs):
prefix = opts.defaults.expand('%{_prefix}')
prefix = macro_expand(opts.defaults, '%{_prefix}')
if opts.canadian_cross():
opts.disable_install()

View File

@ -0,0 +1,75 @@
#
# RTEMS Tools Project (http://www.rtems.org/)
# Copyright 2019 Chris Johns (chrisj@rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
# This code builds a package compiler tool suite given a tool set. A tool
# set lists the various tools. These are specific tool configurations.
#
from __future__ import print_function
try:
import error
import execute
import log
import options
except KeyboardInterrupt:
print('abort: user terminated', file = sys.stderr)
sys.exit(1)
except:
print('error: unknown application load error', file = sys.stderr)
sys.exit(1)
def expand(macros, line):
#
# Parse the line and handle nesting '()' pairs.
#
def _exec(shell_macro):
output = ''
if len(shell_macro) > 3:
e = execute.capture_execution()
if options.host_windows:
cmd = '%s -c "%s"' % (macros.expand('%{__sh}'), shell_macro[2:-1])
else:
cmd = shell_macro[2:-1]
exit_code, proc, output = e.shell(cmd)
log.trace('shell-output: %d %s' % (exit_code, output))
if exit_code != 0:
raise error.general('shell macro failed: %s: %d: %s' % (cmd,
exit_code,
output))
return output
updating = True
while updating:
updating = False
pos = line.find('%(')
if pos >= 0:
braces = 0
for p in range(pos + 2, len(line)):
if line[p] == '(':
braces += 1
elif line[p] == ')':
if braces > 0:
braces -= 1
else:
line = line[:pos] + _exec(line[pos:p + 1]) + line[p + 1:]
updating = True
break
return line