rtems/rtems-source-builder
1
0
Fork 0
mirror of https://git.rtems.org/rtems-source-builder synced 2024-10-09 07:15:10 +08:00
Code Issues Packages Projects Releases Wiki Activity
rtems-source-builder/source-builder/sb/build.py
Chris Johns cb12e4875c Refactor defaults, macros and options. 
To support building snapshots and pre-release source the defaults
has been refactored. The defaults have been moved to a stand alone
file and a macros.py module added. This modile abstracts the
old default dictionary turning it into a class. The macros
class can load macros from a file therefore the defaults have
been moved to a stand alone file.

The use of defaults has been removed from the project. The only
case where it is used in the options where the defaults are read
from a file. Macros are used everywhere now.

The defaults.py has been moved to the option.py and the separate
options and defaults values has been moved to a new pattern. When
constructing an object that needs macros and options if the macros
passed in is None the defaults from the options are used. This makes
it clear when the defaults are being used or when a modified set of
macros is being used.

The macros class support maps. The default is 'global' and where all
the defaults reside and where configuratiion file changes end up.
Maps allow macros to be read from a file and override the values
being maintained in the 'global' map. Reading a macro first checks
the map and if not present checks the 'global' map.

The addition of maps to the macros provides the base to support
snapshots and pre-release testing with standard configurations.
This functionality needs to be added. It works by letting to
specify a snapshot with:

source0: none, override, 'my-dist.tar.bz2'

and it will be used rather the value from the standard configuration.
With a build set you need to also specify the package these macros
are for. The maps provide this.
2013-04-09 13:51:43 +10:00

560 lines
20 KiB
Python
Raw Blame History

#
# RTEMS Tools Project (http://www.rtems.org/)
# Copyright 2010-2012 Chris Johns (chrisj@rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
# This code builds a package given a config file. It only builds to be
# installed not to be package unless you run a packager around this.
#
import getopt
import glob
import os
import shutil
import stat
import sys
import urllib2
import urlparse
try:
import check
import config
import error
import execute
import log
import options
import path
import version
except KeyboardInterrupt:
print 'abort: user terminated'
sys.exit(1)
except:
print 'error: unknown application load error'
sys.exit(1)
def _notice(opts, text):
if not opts.quiet() and not log.default.has_stdout():
print text
log.output(text)
log.flush()
class script:
"""Create and manage a shell script."""
def __init__(self, quiet = True, trace = False):
self.quiet = quiet
self.trace = trace
self.reset()
def reset(self):
self.body = []
self.lc = 0
def append(self, text):
if type(text) is str:
text = text.splitlines()
if not self.quiet:
i = 0
for l in text:
i += 1
log.output('script:%3d: %s' % (self.lc + i, l))
if self.trace:
print '%3d: S %s' % (self.lc + i, l)
self.lc += len(text)
self.body.extend(text)
def write(self, name, check_for_errors = False):
s = None
try:
s = open(path.host(name), 'w')
s.write('\n'.join(self.body))
s.close()
os.chmod(path.host(name), stat.S_IRWXU | \
stat.S_IRGRP | stat.S_IXGRP | \
stat.S_IROTH | stat.S_IXOTH)
except IOError, err:
raise error.general('creating script: ' + name)
except:
if s is not None:
s.close()
raise
if s is not None:
s.close()
class build:
"""Build a package given a config file."""
def __init__(self, name, create_tar_files, opts, macros = None):
self.opts = opts
if macros is None:
self.macros = opts.defaults
else:
self.macros = macros
self.create_tar_files = create_tar_files
_notice(opts, 'config: ' + name)
self.config = config.file(name, opts, self.macros)
self.script = script(quiet = opts.quiet(), trace = opts.trace())
def _output(self, text):
if not self.opts.quiet():
log.output(text)
def rmdir(self, rmpath):
self._output('removing: %s' % (path.host(rmpath)))
if not self.opts.dry_run():
if path.exists(rmpath):
path.removeall(rmpath)
def mkdir(self, mkpath):
self._output('making dir: %s' % (path.host(mkpath)))
if not self.opts.dry_run():
path.mkdir(mkpath)
def get_file(self, url, local):
if local is None:
raise error.general('source/patch path invalid')
if not path.isdir(path.dirname(local)):
_notice(self.opts,
'Creating source directory: %s' % (os.path.relpath(path.host(path.dirname(local)))))
self.mkdir(path.host(path.dirname(local)))
if not path.exists(local):
#
# Not localy found so we need to download it. Check if a URL has
# been provided on the command line.
#
url_bases = self.opts.urls()
urls = []
if url_bases is not None:
for base in url_bases:
if base[-1:] != '/':
base += '/'
url_path = urlparse.urlsplit(url)[2]
slash = url_path.rfind('/')
if slash < 0:
url_file = url_path
else:
url_file = url_path[slash + 1:]
urls.append(urlparse.urljoin(base, url_file))
urls.append(url)
if self.opts.trace():
print '_url:', ','.join(urls), '->', local
for url in urls:
#
# Hack for GitHub.
#
if url.startswith('https://api.github.com'):
url = urlparse.urljoin(url, self.config.expand('tarball/%{version}'))
_notice(self.opts, 'download: %s -> %s' % (url, os.path.relpath(path.host(local))))
if not self.opts.dry_run():
failed = False
_in = None
_out = None
try:
_in = urllib2.urlopen(url)
_out = open(path.host(local), 'wb')
_out.write(_in.read())
except IOError, err:
msg = 'download: %s: error: %s' % (url, str(err))
_notice(self.opts, msg)
if path.exists(local):
os.remove(path.host(local))
failed = True
except ValueError, err:
msg = 'download: %s: error: %s' % (url, str(err))
_notice(self.opts, msg)
if path.exists(local):
os.remove(path.host(local))
failed = True
except:
msg = 'download: %s: error' % (url)
print >> sys.stderr, msg
if _out is not None:
_out.close()
raise
if _out is not None:
_out.close()
if _in is not None:
del _in
if not failed:
if not path.isfile(local):
raise error.general('source is not a file: %s' % (path.host(local)))
return
if not self.opts.dry_run():
raise error.general('downloading %s: all paths have failed, giving up' % (url))
def parse_url(self, url, pathkey):
#
# Split the source up into the parts we need.
#
source = {}
source['url'] = url
source['path'] = path.dirname(url)
source['file'] = path.basename(url)
source['name'], source['ext'] = path.splitext(source['file'])
#
# Get the file. Checks the local source directory first.
#
source['local'] = None
for p in self.config.define(pathkey).split(':'):
local = path.join(path.abspath(p), source['file'])
if source['local'] is None:
source['local'] = local
if path.exists(local):
source['local'] = local
break
#
# Is the file compressed ?
#
esl = source['ext'].split('.')
if esl[-1:][0] == 'gz':
source['compressed'] = '%{__gzip} -dc'
elif esl[-1:][0] == 'bz2':
source['compressed'] = '%{__bzip2} -dc'
elif esl[-1:][0] == 'bz2':
source['compressed'] = '%{__zip} -u'
elif esl[-1:][0] == 'xz':
source['compressed'] = '%{__xz} -dc'
source['script'] = ''
return source
def source(self, package, source_tag):
#
# Scan the sources found in the config file for the one we are
# after. Infos or tags are lists.
#
sources = package.sources()
url = None
for s in sources:
tag = s[len('source'):]
if tag.isdigit():
if int(tag) == source_tag:
url = sources[s][0]
break
if url is None:
raise error.general('source tag not found: source%d' % (source_tag))
source = self.parse_url(url, '_sourcedir')
self.get_file(source['url'], source['local'])
if 'compressed' in source:
source['script'] = source['compressed'] + ' ' + \
source['local'] + ' | %{__tar_extract} -'
else:
source['script'] = '%{__tar_extract} ' + source['local']
return source
def patch(self, package, args):
#
# Scan the patches found in the config file for the one we are
# after. Infos or tags are lists.
#
patches = package.patches()
url = None
for p in patches:
if args[0][1:].lower() == p:
url = patches[p][0]
break
if url is None:
raise error.general('patch tag not found: %s' % (args[0]))
#
# Parse the URL first in the source builder's patch directory.
#
patch = self.parse_url(url, '_patchdir')
#
# If not in the source builder package check the source directory.
#
if not path.exists(patch['local']):
patch = self.parse_url(url, '_patchdir')
self.get_file(patch['url'], patch['local'])
if 'compressed' in patch:
patch['script'] = patch['compressed'] + ' ' + patch['local']
else:
patch['script'] = '%{__cat} ' + patch['local']
patch['script'] += ' | %{__patch} ' + ' '.join(args[1:])
self.script.append(self.config.expand(patch['script']))
def canadian_cross(self):
_host = self.config.expand('%{_host}')
_build = self.config.expand('%{_build}')
_target = self.config.expand('%{_target}')
return self.config.defined('%{allow_cxc}') and \
_host != _build and _host != _target
def setup(self, package, args):
self._output('prep: %s: %s' % (package.name(), ' '.join(args)))
opts, args = getopt.getopt(args[1:], 'qDcTn:b:a:')
source_tag = 0
quiet = False
unpack_default_source = True
unpack_before_chdir = True
delete_before_unpack = True
create_dir = False
name = None
for o in opts:
if o[0] == '-q':
quiet = True
elif o[0] == '-D':
delete_before_unpack = False
elif o[0] == '-c':
create_dir = True
elif o[0] == '-T':
unpack_default_source = False
elif o[0] == '-n':
name = o[1]
elif o[0] == '-b':
unpack_before_chdir = True
if not o[1].isdigit():
raise error.general('setup -b source tag is not a number: %s' % (o[1]))
source_tag = int(o[1])
elif o[0] == '-a':
unpack_before_chdir = False
if not o[1].isdigit():
raise error.general('setup -a source tag is not a number: %s' % (o[1]))
source_tag = int(o[1])
source0 = None
source = self.source(package, source_tag)
if name is None:
if source:
name = source['name']
else:
raise error.general('setup source tag not found: %d' % (source_tag))
self.script.append(self.config.expand('cd %{_builddir}'))
if delete_before_unpack:
self.script.append(self.config.expand('%{__rm} -rf ' + name))
if create_dir:
self.script.append(self.config.expand('%{__mkdir_p} ' + name))
#
# If -a? then change directory before unpacking.
#
if not unpack_before_chdir or create_dir:
self.script.append(self.config.expand('cd ' + name))
#
# Unpacking the source. Note, treated the same as -a0.
#
if unpack_default_source and source_tag != 0:
source0 = self.source(package, 0)
if source0 is None:
raise error.general('no setup source0 tag found')
self.script.append(self.config.expand(source0['script']))
self.script.append(self.config.expand(source['script']))
if unpack_before_chdir and not create_dir:
self.script.append(self.config.expand('cd ' + name))
self.script.append(self.config.expand('%{__setup_post}'))
def run(self, command, shell_opts = '', cwd = None):
e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
self._output('run: ' + cmd)
exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
if exit_code != 0:
raise error.general('shell cmd failed: %s' % (cmd))
def builddir(self):
builddir = self.config.abspath('_builddir')
self.rmdir(builddir)
if not self.opts.dry_run():
self.mkdir(builddir)
def prep(self, package):
self.script.append('echo "==> %prep:"')
_prep = package.prep()
for l in _prep:
args = l.split()
if args[0] == '%setup':
self.setup(package, args)
elif args[0].startswith('%patch'):
self.patch(package, args)
else:
self.script.append(' '.join(args))
def build(self, package):
self.script.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
self.script.append('%s ${SB_BUILD_ROOT}' %
(self.config.expand('%{__rmdir}')))
self.script.append('%s ${SB_BUILD_ROOT}' %
(self.config.expand('%{__mkdir_p}')))
self.script.append('echo "==> %build:"')
_build = package.build()
for l in _build:
args = l.split()
self.script.append(' '.join(args))
def install(self, package):
self.script.append('echo "==> %install:"')
_install = package.install()
for l in _install:
args = l.split()
self.script.append(' '.join(args))
def files(self, package):
if self.create_tar_files:
self.script.append('echo "==> %files:"')
inpath = path.abspath(self.config.expand('%{buildroot}'))
tardir = path.abspath(self.config.expand('%{_tardir}'))
self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
self.script.append(self.config.expand(' %%{__mkdir_p} %s' % tardir))
self.script.append(self.config.expand(' cd ' + inpath))
tar = path.join(tardir, package.long_name() + '.tar.bz2')
cmd = self.config.expand(' %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
self.script.append(cmd)
self.script.append(self.config.expand(' cd %{_builddir}'))
self.script.append('fi')
def clean(self, package):
self.script.append('echo "==> %clean:"')
_clean = package.clean()
if _clean is not None:
for l in _clean:
args = l.split()
self.script.append(' '.join(args))
def build_package(self, package):
if self.canadian_cross():
self.script.append('echo "==> Candian-cross build/target:"')
self.script.append('SB_CXC="yes"')
else:
self.script.append('SB_CXC="no"')
self.build(package)
self.install(package)
self.files(package)
if not self.opts.no_clean():
self.clean(package)
def cleanup(self):
if not self.opts.no_clean():
buildroot = self.config.abspath('buildroot')
builddir = self.config.abspath('_builddir')
buildcxcdir = self.config.abspath('_buildcxcdir')
tmproot = self.config.abspath('_tmproot')
if self.opts.trace():
_notice(self.opts, 'cleanup: %s' % (buildroot))
self.rmdir(buildroot)
if self.opts.trace():
_notice(self.opts, 'cleanup: %s' % (builddir))
self.rmdir(builddir)
if self.canadian_cross():
if self.opts.trace():
_notice(self.opts, 'cleanup: %s' % (buildcxcdir))
self.rmdir(buildcxcdir)
if self.opts.trace():
_notice(self.opts, 'cleanup: %s' % (tmproot))
self.rmdir(tmproot)
def main_package(self):
packages = self.config.packages()
return packages['main']
def make(self):
package = self.main_package()
name = package.name()
if self.canadian_cross():
_notice(self.opts, 'package: (Cxc) %s' % (name))
else:
_notice(self.opts, 'package: %s' % (name))
self.script.reset()
self.script.append(self.config.expand('%{___build_template}'))
self.script.append('echo "=> ' + name + ':"')
self.prep(package)
self.build_package(package)
if not self.opts.dry_run():
self.builddir()
sn = path.join(self.config.expand('%{_builddir}'), 'doit')
self._output('write script: ' + sn)
self.script.write(sn)
if self.canadian_cross():
_notice(self.opts, 'building: (Cxc) %s' % (name))
else:
_notice(self.opts, 'building: %s' % (name))
self.run(sn)
def name(self):
packages = self.config.packages()
package = packages['main']
return package.name()
def get_configs(opts):
def _scan(_path, ext):
configs = []
for root, dirs, files in os.walk(_path):
prefix = root[len(_path) + 1:]
for file in files:
for e in ext:
if file.endswith(e):
configs += [path.join(prefix, file)]
return configs
configs = { 'paths': [], 'files': [] }
for cp in opts.defaults.expand('%{_configdir}').split(':'):
hcp = path.host(path.abspath(cp))
configs['paths'] += [hcp]
configs['files'] += _scan(hcp, ['.cfg', '.bset'])
configs['files'] = sorted(configs['files'])
return configs
def find_config(config, configs):
config_root, config_ext = path.splitext(config)
if config_ext not in ['', '.bset', '.cfg']:
config_root = config
config_ext = ''
for c in configs['files']:
r, e = path.splitext(c)
if config_root == r:
if config_ext == '' or config_ext == e:
return c
return None
def run(args):
try:
optargs = { '--list-configs': 'List available configurations' }
opts = options.load(args, optargs)
log.default = log.log(opts.logfiles())
_notice(opts, 'RTEMS Source Builder, Package Builder v%s' % (version.str()))
if not check.host_setup(opts):
if not opts.force():
raise error.general('host build environment is not set up' +
' correctly (use --force to proceed)')
_notice(opts, 'warning: forcing build with known host setup problems')
if opts.get_arg('--list-configs'):
configs = get_configs(opts)
for p in configs['paths']:
print 'Examining: %s' % (os.path.relpath(p))
for c in configs['files']:
if c.endswith('.cfg'):
print ' %s' % (c)
else:
for config_file in opts.config_files():
b = build(config_file, True, opts)
b.make()
del b
except error.general, gerr:
print gerr
print >> sys.stderr, 'Build FAILED'
sys.exit(1)
except error.internal, ierr:
print ierr
print >> sys.stderr, 'Build FAILED'
sys.exit(1)
except error.exit, eerr:
pass
except KeyboardInterrupt:
_notice(opts, 'abort: user terminated')
sys.exit(1)
sys.exit(0)
if __name__ == "__main__":
run(sys.argv)
Reference in New Issue View Git Blame Copy Permalink