rtems/rtems-source-builder
1
0
Fork 0
mirror of https://git.rtems.org/rtems-source-builder synced 2024-10-09 07:15:10 +08:00
Code Issues Packages Projects Releases Wiki Activity
rtems-source-builder/source-builder/sb/build.py
Chris Johns 6116d9c872 sb: Back port fixes from the development branch 
Updates #4716
2022-09-08 15:47:40 +10:00

719 lines
28 KiB
Python
Raw Permalink Blame History

#
# RTEMS Tools Project (http://www.rtems.org/)
# Copyright 2010-2018 Chris Johns (chrisj@rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
# This code builds a package given a config file. It only builds to be
# installed not to be package unless you run a packager around this.
#
from __future__ import print_function
import copy
import getopt
import glob
import os
import shutil
import stat
import sys
try:
import check
import config
import download
import error
import ereport
import execute
import log
import options
import path
import sources
import version
except KeyboardInterrupt:
print('abort: user terminated')
sys.exit(1)
except:
raise
def humanize_number(num, suffix):
for unit in ['','K','M','G','T','P','E','Z']:
if abs(num) < 1024.0:
return "%5.3f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.3f%s%s" % (size, 'Y', suffix)
def short_name(name):
#
# If on Windows use short names to keep the build paths as short as possible.
#
if options.host_windows:
buildname = ''
add = True
for n in name.split('-'):
buildname += n[0]
return buildname
else:
return name
class script:
"""Create and manage a shell script."""
def __init__(self):
self.reset()
def reset(self):
self.body = []
self.lc = 0
def append(self, text):
is_str = False
if type(text) is str:
is_str = True
try:
if type(text) is unicode:
is_str = True
except:
pass
if is_str:
text = text.splitlines()
if not log.quiet:
i = 0
for l in text:
i += 1
log.output('script:%3d: %s' % (self.lc + i, l))
self.lc += len(text)
self.body.extend(text)
def write(self, name, check_for_errors = False):
s = None
try:
s = open(path.host(name), 'w')
s.write('\n'.join(self.body))
s.close()
os.chmod(path.host(name), stat.S_IRWXU | \
stat.S_IRGRP | stat.S_IXGRP | \
stat.S_IROTH | stat.S_IXOTH)
except IOError as err:
raise error.general('creating script: ' + name)
except:
if s is not None:
s.close()
raise
if s is not None:
s.close()
class build:
"""Build a package given a config file."""
def _generate_report_(self, header, footer = None):
ereport.generate('rsb-report-%s.txt' % self.macros['name'],
self.opts, header, footer)
def __init__(self, name, create_tar_files, opts, macros = None):
try:
self.opts = opts
self.init_name = name
self.init_macros = macros
self.config = None
self.create_tar_files = create_tar_files
log.notice('config: ' + name)
self.set_macros(macros)
self.config = config.file(name, opts, self.macros)
self.script_build = script()
self.script_clean = script()
self.macros['buildname'] = short_name(self.macros['name'])
except error.general as gerr:
log.notice(str(gerr))
log.stderr('Build FAILED')
raise
except error.internal as ierr:
log.notice(str(ierr))
log.stderr('Internal Build FAILED')
raise
except:
raise
def copy_init_macros(self):
return copy.copy(self.init_macros)
def copy_macros(self):
return copy.copy(self.macros)
def set_macros(self, macros):
if macros is None:
self.macros = copy.copy(self.opts.defaults)
else:
self.macros = copy.copy(macros)
if self.config:
self.config.set_macros(self.macros)
def rmdir(self, rmpath):
log.output('removing: %s' % (path.host(rmpath)))
if not self.opts.dry_run():
if path.exists(rmpath):
path.removeall(rmpath)
def mkdir(self, mkpath):
log.output('making dir: %s' % (path.host(mkpath)))
if not self.opts.dry_run():
path.mkdir(mkpath)
def canadian_cross(self):
_host = self.config.expand('%{_host}')
_build = self.config.expand('%{_build}')
_target = self.config.expand('%{_target}')
_allowed = self.config.defined('%{allow_cxc}')
if len(_host) and len(_build) and (_target) and \
_allowed and _host != _build and _host != _target:
return True
return False
def installable(self):
_host = self.config.expand('%{_host}')
_build = self.config.expand('%{_build}')
_canadian_cross = self.canadian_cross()
if self.macros.get('_disable_installing') and \
self.config.expand('%{_disable_installing}') == 'yes':
_disable_installing = True
else:
_disable_installing = False
_no_install = self.opts.no_install()
log.trace('_build: installable: host=%s build=%s ' \
'no-install=%r Cxc=%r disable_installing=%r disabled=%r' % \
(_host, _build, _no_install, _canadian_cross, _disable_installing, \
self.disabled()))
return len(_host) and len(_build) and \
not self.disabled() and \
not _disable_installing and \
not _canadian_cross
def source(self, name, strip_components, download_only):
#
# Return the list of sources. Merge in any macro defined sources as
# these may be overridden by user loaded macros.
#
_map = 'source-%s' % (name)
src_keys = [s for s in self.macros.map_keys(_map) if s != 'setup']
if len(src_keys) == 0:
raise error.general('no source set: %s (%s)' % (name, _map))
srcs = []
for s in src_keys:
sm = self.macros.get(s, globals = False, maps = _map)
if sm is None:
raise error.internal('source macro not found: %s in %s (%s)' % \
(s, name, _map))
opts = []
url = []
for sp in sm[2].split():
if len(url) == 0 and sp[0] == '-':
opts += [sp]
else:
url += [sp]
if len(url) == 0:
raise error.general('source URL not found: %s' % (' '.join(args)))
#
# Look for --rsb-file as an option we use as a local file name.
# This can be used if a URL has no reasonable file name the
# download URL parser can figure out.
#
file_override = None
if len(opts) > 0:
for o in opts:
if o.startswith('--rsb-file'):
os_ = o.split('=')
if len(os_) != 2:
raise error.general('invalid --rsb-file option: %s' % \
(' '.join(args)))
if os_[0] != '--rsb-file':
raise error.general('invalid --rsb-file option: %s' % \
(' '.join(args)))
file_override = os_[1]
opts = [o for o in opts if not o.startswith('--rsb-')]
url = self.config.expand(' '.join(url))
src = download.parse_url(url, '_sourcedir',
self.config, self.opts, file_override)
download.get_file(src['url'], src['local'], self.opts, self.config)
if not download_only:
if strip_components > 0:
tar_extract = '%%{__tar_extract} --strip-components %d' % \
(strip_components)
else:
tar_extract = '%{__tar_extract}'
if 'symlink' in src:
sname = name.replace('-', '_')
src['script'] = '%%{__ln_s} %s ${source_dir_%s}' % \
(src['symlink'], sname)
elif 'compressed' in src:
#
# Zip files unpack as well so do not use tar.
#
src['script'] = '%s %s' % (src['compressed'], src['local'])
if src['compressed-type'] != 'zip':
src['script'] += ' | %s -f -' % (tar_extract)
else:
src['script'] = '%s -f %s' % (tar_extract, src['local'])
srcs += [src]
return srcs
def source_setup(self, package, args):
log.output('source setup: %s: %s' % (package.name(), ' '.join(args)))
setup_name = args[1]
args = args[1:]
try:
opts, args = getopt.getopt(args[1:], 'qDcn:bas:gE')
except getopt.GetoptError as ge:
raise error.general('source setup error: %s' % str(ge))
quiet = False
unpack_before_chdir = True
delete_before_unpack = True
create_dir = False
deleted_dir = False
created_dir = False
changed_dir = False
no_errors = False
strip_components = 0
opt_name = None
download_only = False
for o in opts:
if o[0] == '-q':
quiet = True
elif o[0] == '-D':
delete_before_unpack = False
elif o[0] == '-c':
create_dir = True
elif o[0] == '-n':
opt_name = o[1]
elif o[0] == '-b':
unpack_before_chdir = True
elif o[0] == '-a':
unpack_before_chdir = False
elif o[0] == '-E':
no_errors = True
elif o[0] == '-s':
if not o[1].isdigit():
raise error.general('source setup error: invalid strip count: %s' % \
(o[1]))
strip_components = int(o[1])
elif o[0] == '-g':
download_only = True
name = None
for source in self.source(setup_name, strip_components, download_only):
if name is None:
if opt_name is None:
if source:
opt_name = source['name']
else:
raise error.general('setup source tag not found: %d' % \
(source_tag))
else:
name = opt_name
if not download_only:
self.script_build.append(self.config.expand('cd %{_builddir}'))
if not deleted_dir and delete_before_unpack and name is not None:
self.script_build.append(self.config.expand('%{__rm} -rf ' + name))
deleted_dir = True
if not created_dir and create_dir and name is not None:
self.script_build.append(self.config.expand('%{__mkdir_p} ' + name))
created_dir = True
if not changed_dir and (not unpack_before_chdir or create_dir) and \
name is not None:
self.script_build.append(self.config.expand('cd ' + name))
changed_dir = True
#
# On Windows tar can fail on links if the link appears in the
# tar file before the target of the link exists. We can assume the
# tar file is correct, that is all files and links are valid,
# so on error redo the untar a second time.
#
if options.host_windows or no_errors:
self.script_build.append('set +e')
self.script_build.append(self.config.expand(source['script']))
if options.host_windows or not no_errors:
self.script_build.append('tar_exit=$?')
if options.host_windows or no_errors:
self.script_build.append('set -e')
if options.host_windows:
if no_errors:
self.script_build.append(' set +e')
self.script_build.append(' ' + self.config.expand(source['script']))
self.script_build.append(' set -e')
else:
self.script_build.append('if test $tar_exit != 0; then')
self.script_build.append(' ' + self.config.expand(source['script']))
self.script_build.append('fi')
if not changed_dir and (unpack_before_chdir and not create_dir) and \
name is not None and not download_only:
self.script_build.append(self.config.expand('cd ' + name))
changed_dir = True
self.script_build.append(self.config.expand('%{__setup_post}'))
def patch_setup(self, package, args):
name = args[1]
args = args[2:]
_map = 'patch-%s' % (name)
default_opts = ' '.join(args)
patch_keys = [p for p in self.macros.map_keys(_map) if p != 'setup']
patches = []
for p in patch_keys:
pm = self.macros.get(p, globals = False, maps = _map)
if pm is None:
raise error.internal('patch macro not found: %s in %s (%s)' % \
(p, name, _map))
opts = []
url = []
for pp in pm[2].split():
if len(url) == 0 and pp[0] == '-':
opts += [pp]
else:
url += [pp]
if len(url) == 0:
raise error.general('patch URL not found: %s' % (' '.join(opts)))
#
# Look for --rsb-file as an option we use as a local file name.
# This can be used if a URL has no reasonable file name the
# download URL parser can figure out.
#
file_override = None
if len(opts) > 0:
for o in opts:
if o.startswith('--rsb-file'):
os_ = o.split('=')
if len(os_) != 2:
raise error.general('invalid --rsb-file option: %s' % \
(' '.join(opts)))
if os_[0] != '--rsb-file':
raise error.general('invalid --rsb-file option: %s' % \
(' '.join(opts)))
file_override = os_[1]
opts = [o for o in opts if not o.startswith('--rsb-')]
if len(opts) == 0:
opts = default_opts
else:
opts = ' '.join(opts)
opts = self.config.expand(opts)
url = self.config.expand(' '.join(url))
#
# Parse the URL first in the source builder's patch directory.
#
patch = download.parse_url(url, '_patchdir', self.config,
self.opts, file_override)
#
# Download the patch
#
download.get_file(patch['url'], patch['local'], self.opts, self.config)
if 'compressed' in patch:
patch['script'] = patch['compressed'] + ' ' + patch['local']
else:
patch['script'] = '%{__cat} ' + patch['local']
patch['script'] += ' | %%{__patch} %s' % (opts)
self.script_build.append(self.config.expand(patch['script']))
def run(self, command, shell_opts = '', cwd = None):
e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
log.output('run: ' + cmd)
exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
if exit_code != 0:
log.output('shell cmd failed: %s' % (cmd))
raise error.general('building %s' % (self.macros['buildname']))
def builddir(self):
builddir = self.config.abspath('_builddir')
self.rmdir(builddir)
if not self.opts.dry_run():
self.mkdir(builddir)
def prep(self, package):
self.script_build.append('echo "==> %prep:"')
_prep = package.prep()
if _prep:
for l in _prep:
args = l.split()
if len(args):
def err(msg):
raise error.general('%s: %s' % (package, msg))
if args[0] == '%setup':
if len(args) == 1:
raise error.general('invalid %%setup directive: %s' % \
(' '.join(args)))
if args[1] == 'source':
self.source_setup(package, args[1:])
elif args[1] == 'patch':
self.patch_setup(package, args[1:])
elif args[0] in ['%patch', '%source']:
sources.process(args[0][1:], args[1:], self.macros, err)
elif args[0] == '%hash':
sources.hash(args[1:], self.macros, err)
self.hash(package, args)
else:
self.script_build.append(' '.join(args))
def build(self, package):
self.script_build.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
self.script_build.append('%s ${SB_BUILD_ROOT}' %
(self.config.expand('%{__rmdir}')))
self.script_build.append('%s ${SB_BUILD_ROOT}' %
(self.config.expand('%{__mkdir_p}')))
self.script_build.append('echo "==> %build:"')
_build = package.build()
if _build:
for l in _build:
self.script_build.append(l)
def install(self, package):
self.script_build.append('echo "==> %install:"')
_install = package.install()
if _install:
for l in _install:
args = l.split()
self.script_build.append(' '.join(args))
def files(self, package):
if self.create_tar_files \
and not self.macros.get('%{_disable_packaging'):
self.script_build.append('echo "==> %files:"')
inpath = path.abspath(self.config.expand('%{buildroot}'))
tardir = path.abspath(self.config.expand('%{_tardir}'))
self.script_build.append(self.config.expand('if test -d %s; then' % (inpath)))
self.script_build.append(self.config.expand(' %%{__mkdir_p} %s' % tardir))
self.script_build.append(self.config.expand(' cd ' + inpath))
tar = path.join(tardir, package.long_name() + '.tar.bz2')
cmd = self.config.expand(' %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
self.script_build.append(cmd)
self.script_build.append(self.config.expand(' cd %{_builddir}'))
self.script_build.append('fi')
def clean(self, package):
self.script_clean.reset()
self.script_clean.append(self.config.expand('%{___build_template}'))
self.script_clean.append('echo "=> ' + package.name() + ': CLEAN"')
self.script_clean.append('echo "==> %clean:"')
_clean = package.clean()
if _clean is not None:
for l in _clean:
args = l.split()
self.script_clean.append(' '.join(args))
def sizes(self, package):
def _sizes(package, what, path):
package.set_size(what, path)
s = humanize_number(package.get_size(what), 'B')
log.trace('size: %s (%s): %s (%d)' % (what, path, s, package.get_size(what)))
return s
s = {}
for p in [('build', '%{_builddir}'),
('build', '%{buildroot}'),
('installed', '%{buildroot}')]:
hs = _sizes(package, p[0], self.config.expand(p[1]))
s[p[0]] = hs
log.notice('sizes: %s: %s (installed: %s)' % (package.name(),
s['build'],
s['installed']))
def build_package(self, package):
if self.canadian_cross():
if not self.config.defined('%{allow_cxc}'):
raise error.general('Canadian Cross is not allowed')
self.script_build.append('echo "==> Candian-cross build/target:"')
self.script_build.append('SB_CXC="yes"')
else:
self.script_build.append('SB_CXC="no"')
self.build(package)
self.install(package)
self.files(package)
if not self.opts.no_clean():
self.clean(package)
def cleanup(self):
package = self.main_package()
if not package.disabled() and not self.opts.no_clean():
buildroot = self.config.abspath('buildroot')
builddir = self.config.abspath('_builddir')
buildcxcdir = self.config.abspath('_buildcxcdir')
tmproot = self.config.abspath('_tmproot')
log.trace('cleanup: %s' % (buildroot))
self.rmdir(buildroot)
log.trace('cleanup: %s' % (builddir))
self.rmdir(builddir)
if self.canadian_cross():
log.trace('cleanup: %s' % (buildcxcdir))
self.rmdir(buildcxcdir)
log.trace('cleanup: %s' % (tmproot))
self.rmdir(tmproot)
def main_package(self):
packages = self.config.packages()
return packages['main']
def reload(self):
self.config.load(self.init_name)
def make(self):
package = self.main_package()
if package.disabled():
log.notice('package: nothing to build')
else:
try:
name = package.name()
if self.canadian_cross():
cxc_label = '(Cxc) '
else:
cxc_label = ''
log.notice('package: %s%s' % (cxc_label, name))
log.trace('---- macro maps %s' % ('-' * 55))
log.trace('%s' % (str(self.config.macros)))
log.trace('-' * 70)
self.script_build.reset()
self.script_build.append(self.config.expand('%{___build_template}'))
self.script_build.append('echo "=> ' + name + ': BUILD"')
self.prep(package)
self.build_package(package)
if not self.opts.dry_run():
self.builddir()
build_sn = path.join(self.config.expand('%{_builddir}'), 'do-build')
log.output('write script: ' + build_sn)
self.script_build.write(build_sn)
clean_sn = path.join(self.config.expand('%{_builddir}'), 'do-clean')
log.output('write script: ' + clean_sn)
self.script_clean.write(clean_sn)
log.notice('building: %s%s' % (cxc_label, name))
self.run(build_sn)
self.sizes(package)
log.notice('cleaning: %s%s' % (cxc_label, name))
self.run(clean_sn)
except error.general as gerr:
log.notice(str(gerr))
log.stderr('Build FAILED')
self._generate_report_('Build: %s' % (gerr))
raise
except error.internal as ierr:
log.notice(str(ierr))
log.stderr('Internal Build FAILED')
self._generate_report_('Build: %s' % (ierr))
raise
except:
raise
if self.opts.dry_run():
self._generate_report_('Build: dry run (no actual error)',
'Build: dry run (no actual error)')
def name(self):
packages = self.config.packages()
package = packages['main']
return package.name()
def disabled(self):
packages = self.config.packages()
package = packages['main']
return package.disabled()
def get_build_size(self):
package = self.main_package()
if package.disabled():
return 0
return package.get_size('build')
def get_installed_size(self):
package = self.main_package()
if package.disabled():
return 0
return package.get_size('installed')
def includes(self):
if self.config:
return self.config.includes()
def get_configs(opts):
def _scan(_path, ext):
configs = []
for root, dirs, files in os.walk(_path):
prefix = root[len(_path) + 1:]
for file in files:
for e in ext:
if file.endswith(e):
configs += [path.join(prefix, file)]
return configs
configs = { 'paths': [], 'files': [] }
paths = opts.defaults.expand('%{_configdir}').split(':')
root = path.host(os.path.commonprefix(paths))
configs['root'] = root
configs['localpaths'] = [lp[len(root):] for lp in paths]
for cp in paths:
hcp = path.host(path.abspath(cp))
configs['paths'] += [hcp]
hpconfigs = sorted(set(_scan(hcp, ['.cfg', '.bset'])))
hcplocal = hcp[len(root):]
configs[hcplocal] = [path.join(hcplocal, c) for c in hpconfigs]
configs['files'] += hpconfigs
configs['files'] = sorted(set(configs['files']))
return configs
def find_config(config, configs):
config_root, config_ext = path.splitext(config)
if config_ext not in ['', '.bset', '.cfg']:
config_root = config
config_ext = ''
for c in configs['files']:
r, e = path.splitext(c)
if config_root == r:
if config_ext == '' or config_ext == e:
return c
return None
def run(args):
ec = 0
try:
optargs = { '--list-configs': 'List available configurations' }
opts = options.load(args, optargs)
log.notice('RTEMS Source Builder, Package Builder, %s' % (version.string()))
opts.log_info()
if not check.host_setup(opts):
if not opts.force():
raise error.general('host build environment is not set up' +
' correctly (use --force to proceed)')
log.notice('warning: forcing build with known host setup problems')
if opts.get_arg('--list-configs'):
configs = get_configs(opts)
for p in configs['paths']:
print('Examining: %s' % (os.path.relpath(p)))
for c in configs['files']:
if c.endswith('.cfg'):
print(' %s' % (c))
else:
for config_file in opts.config_files():
b = build(config_file, True, opts)
b.make()
b = None
except error.general as gerr:
log.stderr('Build FAILED')
ec = 1
except error.internal as ierr:
log.stderr('Internal Build FAILED')
ec = 1
except error.exit as eerr:
pass
except KeyboardInterrupt:
log.notice('abort: user terminated')
ec = 1
sys.exit(ec)
if __name__ == "__main__":
run(sys.argv)
Reference in New Issue View Git Blame Copy Permalink