mirror of
https://git.rtems.org/rtems-source-builder
synced 2024-10-09 07:15:10 +08:00
0565e1fa4b
User macro files passed on the command line allow a user to override the defaults in configuration files to test new changes in pending releases. Fix macros issues with keys with more than one map.
565 lines
21 KiB
Python
565 lines
21 KiB
Python
#
|
|
# RTEMS Tools Project (http://www.rtems.org/)
|
|
# Copyright 2010-2012 Chris Johns (chrisj@rtems.org)
|
|
# All rights reserved.
|
|
#
|
|
# This file is part of the RTEMS Tools package in 'rtems-tools'.
|
|
#
|
|
# Permission to use, copy, modify, and/or distribute this software for any
|
|
# purpose with or without fee is hereby granted, provided that the above
|
|
# copyright notice and this permission notice appear in all copies.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
|
|
|
#
|
|
# This code builds a package given a config file. It only builds to be
|
|
# installed not to be package unless you run a packager around this.
|
|
#
|
|
|
|
import getopt
|
|
import glob
|
|
import os
|
|
import shutil
|
|
import stat
|
|
import sys
|
|
import urllib2
|
|
import urlparse
|
|
|
|
try:
|
|
import check
|
|
import config
|
|
import error
|
|
import execute
|
|
import log
|
|
import options
|
|
import path
|
|
import version
|
|
except KeyboardInterrupt:
|
|
print 'abort: user terminated'
|
|
sys.exit(1)
|
|
except:
|
|
print 'error: unknown application load error'
|
|
sys.exit(1)
|
|
|
|
def _notice(opts, text):
|
|
if not opts.quiet() and not log.default.has_stdout():
|
|
print text
|
|
log.output(text)
|
|
log.flush()
|
|
|
|
class script:
|
|
"""Create and manage a shell script."""
|
|
|
|
def __init__(self, quiet = True, trace = False):
|
|
self.quiet = quiet
|
|
self.trace = trace
|
|
self.reset()
|
|
|
|
def reset(self):
|
|
self.body = []
|
|
self.lc = 0
|
|
|
|
def append(self, text):
|
|
if type(text) is str:
|
|
text = text.splitlines()
|
|
if not self.quiet:
|
|
i = 0
|
|
for l in text:
|
|
i += 1
|
|
log.output('script:%3d: %s' % (self.lc + i, l))
|
|
if self.trace:
|
|
print '%3d: S %s' % (self.lc + i, l)
|
|
self.lc += len(text)
|
|
self.body.extend(text)
|
|
|
|
def write(self, name, check_for_errors = False):
|
|
s = None
|
|
try:
|
|
s = open(path.host(name), 'w')
|
|
s.write('\n'.join(self.body))
|
|
s.close()
|
|
os.chmod(path.host(name), stat.S_IRWXU | \
|
|
stat.S_IRGRP | stat.S_IXGRP | \
|
|
stat.S_IROTH | stat.S_IXOTH)
|
|
except IOError, err:
|
|
raise error.general('creating script: ' + name)
|
|
except:
|
|
if s is not None:
|
|
s.close()
|
|
raise
|
|
if s is not None:
|
|
s.close()
|
|
|
|
class build:
|
|
"""Build a package given a config file."""
|
|
|
|
def __init__(self, name, create_tar_files, opts, macros = None):
|
|
self.opts = opts
|
|
if macros is None:
|
|
self.macros = opts.defaults
|
|
else:
|
|
self.macros = macros
|
|
self.create_tar_files = create_tar_files
|
|
_notice(opts, 'config: ' + name)
|
|
self.config = config.file(name, opts, self.macros)
|
|
self.script = script(quiet = opts.quiet(), trace = opts.trace())
|
|
|
|
def _output(self, text):
|
|
if not self.opts.quiet():
|
|
log.output(text)
|
|
|
|
def rmdir(self, rmpath):
|
|
self._output('removing: %s' % (path.host(rmpath)))
|
|
if not self.opts.dry_run():
|
|
if path.exists(rmpath):
|
|
path.removeall(rmpath)
|
|
|
|
def mkdir(self, mkpath):
|
|
self._output('making dir: %s' % (path.host(mkpath)))
|
|
if not self.opts.dry_run():
|
|
path.mkdir(mkpath)
|
|
|
|
def get_file(self, url, local):
|
|
if local is None:
|
|
raise error.general('source/patch path invalid')
|
|
if not path.isdir(path.dirname(local)):
|
|
_notice(self.opts,
|
|
'Creating source directory: %s' % (os.path.relpath(path.host(path.dirname(local)))))
|
|
self.mkdir(path.host(path.dirname(local)))
|
|
if not path.exists(local):
|
|
#
|
|
# Not localy found so we need to download it. Check if a URL has
|
|
# been provided on the command line.
|
|
#
|
|
url_bases = self.opts.urls()
|
|
urls = []
|
|
if url_bases is not None:
|
|
for base in url_bases:
|
|
if base[-1:] != '/':
|
|
base += '/'
|
|
url_path = urlparse.urlsplit(url)[2]
|
|
slash = url_path.rfind('/')
|
|
if slash < 0:
|
|
url_file = url_path
|
|
else:
|
|
url_file = url_path[slash + 1:]
|
|
urls.append(urlparse.urljoin(base, url_file))
|
|
urls.append(url)
|
|
if self.opts.trace():
|
|
print '_url:', ','.join(urls), '->', local
|
|
for url in urls:
|
|
#
|
|
# Hack for GitHub.
|
|
#
|
|
if url.startswith('https://api.github.com'):
|
|
url = urlparse.urljoin(url, self.config.expand('tarball/%{version}'))
|
|
_notice(self.opts, 'download: %s -> %s' % (url, os.path.relpath(path.host(local))))
|
|
if not self.opts.dry_run():
|
|
failed = False
|
|
_in = None
|
|
_out = None
|
|
try:
|
|
_in = urllib2.urlopen(url)
|
|
_out = open(path.host(local), 'wb')
|
|
_out.write(_in.read())
|
|
except IOError, err:
|
|
msg = 'download: %s: error: %s' % (url, str(err))
|
|
_notice(self.opts, msg)
|
|
if path.exists(local):
|
|
os.remove(path.host(local))
|
|
failed = True
|
|
except ValueError, err:
|
|
msg = 'download: %s: error: %s' % (url, str(err))
|
|
_notice(self.opts, msg)
|
|
if path.exists(local):
|
|
os.remove(path.host(local))
|
|
failed = True
|
|
except:
|
|
msg = 'download: %s: error' % (url)
|
|
print >> sys.stderr, msg
|
|
if _out is not None:
|
|
_out.close()
|
|
raise
|
|
if _out is not None:
|
|
_out.close()
|
|
if _in is not None:
|
|
del _in
|
|
if not failed:
|
|
if not path.isfile(local):
|
|
raise error.general('source is not a file: %s' % (path.host(local)))
|
|
return
|
|
if not self.opts.dry_run():
|
|
raise error.general('downloading %s: all paths have failed, giving up' % (url))
|
|
|
|
def parse_url(self, url, pathkey):
|
|
#
|
|
# Split the source up into the parts we need.
|
|
#
|
|
source = {}
|
|
source['url'] = url
|
|
source['path'] = path.dirname(url)
|
|
source['file'] = path.basename(url)
|
|
source['name'], source['ext'] = path.splitext(source['file'])
|
|
#
|
|
# Get the file. Checks the local source directory first.
|
|
#
|
|
source['local'] = None
|
|
for p in self.config.define(pathkey).split(':'):
|
|
local = path.join(path.abspath(p), source['file'])
|
|
if source['local'] is None:
|
|
source['local'] = local
|
|
if path.exists(local):
|
|
source['local'] = local
|
|
break
|
|
#
|
|
# Is the file compressed ?
|
|
#
|
|
esl = source['ext'].split('.')
|
|
if esl[-1:][0] == 'gz':
|
|
source['compressed'] = '%{__gzip} -dc'
|
|
elif esl[-1:][0] == 'bz2':
|
|
source['compressed'] = '%{__bzip2} -dc'
|
|
elif esl[-1:][0] == 'bz2':
|
|
source['compressed'] = '%{__zip} -u'
|
|
elif esl[-1:][0] == 'xz':
|
|
source['compressed'] = '%{__xz} -dc'
|
|
source['script'] = ''
|
|
return source
|
|
|
|
def source(self, package, source_tag):
|
|
#
|
|
# Scan the sources found in the config file for the one we are
|
|
# after. Infos or tags are lists. Merge in any macro defined
|
|
# sources as these may be overridden by user loaded macros.
|
|
#
|
|
sources = package.sources()
|
|
url = None
|
|
for s in sources:
|
|
tag = s[len('source'):]
|
|
if tag.isdigit():
|
|
if int(tag) == source_tag:
|
|
url = sources[s][0]
|
|
break
|
|
if url is None:
|
|
raise error.general('source tag not found: source%d' % (source_tag))
|
|
source = self.parse_url(url, '_sourcedir')
|
|
self.get_file(source['url'], source['local'])
|
|
if 'compressed' in source:
|
|
source['script'] = source['compressed'] + ' ' + \
|
|
source['local'] + ' | %{__tar_extract} -'
|
|
else:
|
|
source['script'] = '%{__tar_extract} ' + source['local']
|
|
return source
|
|
|
|
def patch(self, package, args):
|
|
#
|
|
# Scan the patches found in the config file for the one we are
|
|
# after. Infos or tags are lists.
|
|
#
|
|
patches = package.patches()
|
|
url = None
|
|
for p in patches:
|
|
if args[0][1:].lower() == p:
|
|
url = patches[p][0]
|
|
break
|
|
if url is None:
|
|
raise error.general('patch tag not found: %s' % (args[0]))
|
|
#
|
|
# Parse the URL first in the source builder's patch directory.
|
|
#
|
|
patch = self.parse_url(url, '_patchdir')
|
|
#
|
|
# If not in the source builder package check the source directory.
|
|
#
|
|
if not path.exists(patch['local']):
|
|
patch = self.parse_url(url, '_patchdir')
|
|
self.get_file(patch['url'], patch['local'])
|
|
if 'compressed' in patch:
|
|
patch['script'] = patch['compressed'] + ' ' + patch['local']
|
|
else:
|
|
patch['script'] = '%{__cat} ' + patch['local']
|
|
patch['script'] += ' | %{__patch} ' + ' '.join(args[1:])
|
|
self.script.append(self.config.expand(patch['script']))
|
|
|
|
def canadian_cross(self):
|
|
_host = self.config.expand('%{_host}')
|
|
_build = self.config.expand('%{_build}')
|
|
_target = self.config.expand('%{_target}')
|
|
return self.config.defined('%{allow_cxc}') and \
|
|
_host != _build and _host != _target
|
|
|
|
def setup(self, package, args):
|
|
self._output('prep: %s: %s' % (package.name(), ' '.join(args)))
|
|
opts, args = getopt.getopt(args[1:], 'qDcTn:b:a:')
|
|
source_tag = 0
|
|
quiet = False
|
|
unpack_default_source = True
|
|
unpack_before_chdir = True
|
|
delete_before_unpack = True
|
|
create_dir = False
|
|
name = None
|
|
for o in opts:
|
|
if o[0] == '-q':
|
|
quiet = True
|
|
elif o[0] == '-D':
|
|
delete_before_unpack = False
|
|
elif o[0] == '-c':
|
|
create_dir = True
|
|
elif o[0] == '-T':
|
|
unpack_default_source = False
|
|
elif o[0] == '-n':
|
|
name = o[1]
|
|
elif o[0] == '-b':
|
|
unpack_before_chdir = True
|
|
if not o[1].isdigit():
|
|
raise error.general('setup -b source tag is not a number: %s' % (o[1]))
|
|
source_tag = int(o[1])
|
|
elif o[0] == '-a':
|
|
unpack_before_chdir = False
|
|
if not o[1].isdigit():
|
|
raise error.general('setup -a source tag is not a number: %s' % (o[1]))
|
|
source_tag = int(o[1])
|
|
source0 = None
|
|
source = self.source(package, source_tag)
|
|
if name is None:
|
|
if source:
|
|
name = source['name']
|
|
else:
|
|
raise error.general('setup source tag not found: %d' % (source_tag))
|
|
self.script.append(self.config.expand('cd %{_builddir}'))
|
|
if delete_before_unpack:
|
|
self.script.append(self.config.expand('%{__rm} -rf ' + name))
|
|
if create_dir:
|
|
self.script.append(self.config.expand('%{__mkdir_p} ' + name))
|
|
#
|
|
# If -a? then change directory before unpacking.
|
|
#
|
|
if not unpack_before_chdir or create_dir:
|
|
self.script.append(self.config.expand('cd ' + name))
|
|
#
|
|
# Unpacking the source. Note, treated the same as -a0.
|
|
#
|
|
if unpack_default_source and source_tag != 0:
|
|
source0 = self.source(package, 0)
|
|
if source0 is None:
|
|
raise error.general('no setup source0 tag found')
|
|
self.script.append(self.config.expand(source0['script']))
|
|
self.script.append(self.config.expand(source['script']))
|
|
if unpack_before_chdir and not create_dir:
|
|
self.script.append(self.config.expand('cd ' + name))
|
|
self.script.append(self.config.expand('%{__setup_post}'))
|
|
|
|
def run(self, command, shell_opts = '', cwd = None):
|
|
e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
|
|
cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
|
|
self._output('run: ' + cmd)
|
|
exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
|
|
if exit_code != 0:
|
|
raise error.general('shell cmd failed: %s' % (cmd))
|
|
|
|
def builddir(self):
|
|
builddir = self.config.abspath('_builddir')
|
|
self.rmdir(builddir)
|
|
if not self.opts.dry_run():
|
|
self.mkdir(builddir)
|
|
|
|
def prep(self, package):
|
|
self.script.append('echo "==> %prep:"')
|
|
_prep = package.prep()
|
|
for l in _prep:
|
|
args = l.split()
|
|
if args[0] == '%setup':
|
|
self.setup(package, args)
|
|
elif args[0].startswith('%patch'):
|
|
self.patch(package, args)
|
|
else:
|
|
self.script.append(' '.join(args))
|
|
|
|
def build(self, package):
|
|
self.script.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
|
|
self.script.append('%s ${SB_BUILD_ROOT}' %
|
|
(self.config.expand('%{__rmdir}')))
|
|
self.script.append('%s ${SB_BUILD_ROOT}' %
|
|
(self.config.expand('%{__mkdir_p}')))
|
|
self.script.append('echo "==> %build:"')
|
|
_build = package.build()
|
|
for l in _build:
|
|
args = l.split()
|
|
self.script.append(' '.join(args))
|
|
|
|
def install(self, package):
|
|
self.script.append('echo "==> %install:"')
|
|
_install = package.install()
|
|
for l in _install:
|
|
args = l.split()
|
|
self.script.append(' '.join(args))
|
|
|
|
def files(self, package):
|
|
if self.create_tar_files:
|
|
self.script.append('echo "==> %files:"')
|
|
inpath = path.abspath(self.config.expand('%{buildroot}'))
|
|
tardir = path.abspath(self.config.expand('%{_tardir}'))
|
|
self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
|
|
self.script.append(self.config.expand(' %%{__mkdir_p} %s' % tardir))
|
|
self.script.append(self.config.expand(' cd ' + inpath))
|
|
tar = path.join(tardir, package.long_name() + '.tar.bz2')
|
|
cmd = self.config.expand(' %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
|
|
self.script.append(cmd)
|
|
self.script.append(self.config.expand(' cd %{_builddir}'))
|
|
self.script.append('fi')
|
|
|
|
def clean(self, package):
|
|
self.script.append('echo "==> %clean:"')
|
|
_clean = package.clean()
|
|
if _clean is not None:
|
|
for l in _clean:
|
|
args = l.split()
|
|
self.script.append(' '.join(args))
|
|
|
|
def build_package(self, package):
|
|
if self.canadian_cross():
|
|
self.script.append('echo "==> Candian-cross build/target:"')
|
|
self.script.append('SB_CXC="yes"')
|
|
else:
|
|
self.script.append('SB_CXC="no"')
|
|
self.build(package)
|
|
self.install(package)
|
|
self.files(package)
|
|
if not self.opts.no_clean():
|
|
self.clean(package)
|
|
|
|
def cleanup(self):
|
|
if not self.opts.no_clean():
|
|
buildroot = self.config.abspath('buildroot')
|
|
builddir = self.config.abspath('_builddir')
|
|
buildcxcdir = self.config.abspath('_buildcxcdir')
|
|
tmproot = self.config.abspath('_tmproot')
|
|
if self.opts.trace():
|
|
_notice(self.opts, 'cleanup: %s' % (buildroot))
|
|
self.rmdir(buildroot)
|
|
if self.opts.trace():
|
|
_notice(self.opts, 'cleanup: %s' % (builddir))
|
|
self.rmdir(builddir)
|
|
if self.canadian_cross():
|
|
if self.opts.trace():
|
|
_notice(self.opts, 'cleanup: %s' % (buildcxcdir))
|
|
self.rmdir(buildcxcdir)
|
|
if self.opts.trace():
|
|
_notice(self.opts, 'cleanup: %s' % (tmproot))
|
|
self.rmdir(tmproot)
|
|
|
|
def main_package(self):
|
|
packages = self.config.packages()
|
|
return packages['main']
|
|
|
|
def make(self):
|
|
package = self.main_package()
|
|
name = package.name()
|
|
if self.canadian_cross():
|
|
_notice(self.opts, 'package: (Cxc) %s' % (name))
|
|
else:
|
|
_notice(self.opts, 'package: %s' % (name))
|
|
if self.opts.trace():
|
|
print '---- macro maps', '-' * 55
|
|
print self.config.macros
|
|
print '-' * 70
|
|
self.script.reset()
|
|
self.script.append(self.config.expand('%{___build_template}'))
|
|
self.script.append('echo "=> ' + name + ':"')
|
|
self.prep(package)
|
|
self.build_package(package)
|
|
if not self.opts.dry_run():
|
|
self.builddir()
|
|
sn = path.join(self.config.expand('%{_builddir}'), 'doit')
|
|
self._output('write script: ' + sn)
|
|
self.script.write(sn)
|
|
if self.canadian_cross():
|
|
_notice(self.opts, 'building: (Cxc) %s' % (name))
|
|
else:
|
|
_notice(self.opts, 'building: %s' % (name))
|
|
self.run(sn)
|
|
|
|
def name(self):
|
|
packages = self.config.packages()
|
|
package = packages['main']
|
|
return package.name()
|
|
|
|
def get_configs(opts):
|
|
|
|
def _scan(_path, ext):
|
|
configs = []
|
|
for root, dirs, files in os.walk(_path):
|
|
prefix = root[len(_path) + 1:]
|
|
for file in files:
|
|
for e in ext:
|
|
if file.endswith(e):
|
|
configs += [path.join(prefix, file)]
|
|
return configs
|
|
|
|
configs = { 'paths': [], 'files': [] }
|
|
for cp in opts.defaults.expand('%{_configdir}').split(':'):
|
|
hcp = path.host(path.abspath(cp))
|
|
configs['paths'] += [hcp]
|
|
configs['files'] += _scan(hcp, ['.cfg', '.bset'])
|
|
configs['files'] = sorted(configs['files'])
|
|
return configs
|
|
|
|
def find_config(config, configs):
|
|
config_root, config_ext = path.splitext(config)
|
|
if config_ext not in ['', '.bset', '.cfg']:
|
|
config_root = config
|
|
config_ext = ''
|
|
for c in configs['files']:
|
|
r, e = path.splitext(c)
|
|
if config_root == r:
|
|
if config_ext == '' or config_ext == e:
|
|
return c
|
|
return None
|
|
|
|
def run(args):
|
|
try:
|
|
optargs = { '--list-configs': 'List available configurations' }
|
|
opts = options.load(args, optargs)
|
|
log.default = log.log(opts.logfiles())
|
|
_notice(opts, 'RTEMS Source Builder, Package Builder v%s' % (version.str()))
|
|
if not check.host_setup(opts):
|
|
if not opts.force():
|
|
raise error.general('host build environment is not set up' +
|
|
' correctly (use --force to proceed)')
|
|
_notice(opts, 'warning: forcing build with known host setup problems')
|
|
if opts.get_arg('--list-configs'):
|
|
configs = get_configs(opts)
|
|
for p in configs['paths']:
|
|
print 'Examining: %s' % (os.path.relpath(p))
|
|
for c in configs['files']:
|
|
if c.endswith('.cfg'):
|
|
print ' %s' % (c)
|
|
else:
|
|
for config_file in opts.config_files():
|
|
b = build(config_file, True, opts)
|
|
b.make()
|
|
del b
|
|
except error.general, gerr:
|
|
print gerr
|
|
print >> sys.stderr, 'Build FAILED'
|
|
sys.exit(1)
|
|
except error.internal, ierr:
|
|
print ierr
|
|
print >> sys.stderr, 'Build FAILED'
|
|
sys.exit(1)
|
|
except error.exit, eerr:
|
|
pass
|
|
except KeyboardInterrupt:
|
|
_notice(opts, 'abort: user terminated')
|
|
sys.exit(1)
|
|
sys.exit(0)
|
|
|
|
if __name__ == "__main__":
|
|
run(sys.argv)
|