Chris Johns 7d3350d0bb rtemstoolkit: Move host support access into a separate module.
Moving the host support into a module lets it get used where options
is not being used.
2017-04-25 00:31:44 +10:00

875 lines
32 KiB
Python

#
# RTEMS Tools Project (http://www.rtems.org/)
# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
#
# This code is based on a tool I wrote to parse RPM spec files in the RTEMS
# project. This is now a configuration file format that has moved away from the
# spec file format to support the specific needs of cross-compiling GCC. This
# module parses a configuration file into Python data types that can be used by
# other software modules.
#
from __future__ import print_function
import copy
import functools
import os
import re
import sys
#
# Support to handle use in a package and as a unit test.
# If there is a better way to let us know.
#
try:
from . import error
from . import execute
from . import host
from . import log
from . import options
from . import path
except (ValueError, SystemError):
import error
import execute
import host
import log
import options
import path
def _check_bool(value):
if value.isdigit():
if int(value) == 0:
istrue = False
else:
istrue = True
else:
istrue = None
return istrue
class file(object):
"""Parse a config file."""
def __init__(self, name, opts, macros = None, directives = None, ignores = None):
self.opts = opts
if macros is None:
self.macros = opts.defaults
else:
self.macros = macros
self.init_name = name
self.directives = ['%include']
if directives:
self.directives += directives
self.ignores = ignores
log.trace('config: %s' % (name))
self.disable_macro_reassign = False
self.configpath = []
self.wss = re.compile(r'\s+')
self.tags = re.compile(r':+')
self.sf = re.compile(r'%\([^\)]+\)')
for arg in self.opts.args:
if arg.startswith('--with-') or arg.startswith('--without-'):
label = arg[2:].lower().replace('-', '_')
self.macros.define(label)
self._includes = []
self.load_depth = 0
self.lc = 0
self.name = 'none'
def __del__(self):
pass
def __str__(self):
def _dict(dd):
s = ''
ddl = list(dd.keys())
ddl.sort()
for d in ddl:
s += ' ' + d + ': ' + dd[d] + '\n'
return s
s = 'config: %s' % ('.'.join(self.configpath)) + \
'\n' + str(self.opts) + \
'\nlines parsed: %d' % (self.lc) + \
'\nname: ' + self.name + \
'\nmacros:\n' + str(self.macros)
return s
def _name_line_msg(self, msg):
return '%s:%d: %s' % (path.basename(self.init_name), self.lc, msg)
def _output(self, text):
if not self.opts.quiet():
log.output(text)
def _error(self, msg):
err = 'error: %s' % (self._name_line_msg(msg))
log.stderr(err)
log.output(err)
self.in_error = True
if not self.opts.dry_run():
log.stderr('warning: switched to dry run due to errors')
self.opts.set_dry_run()
def _label(self, name):
if name.startswith('%{') and name[-1] is '}':
return name
return '%{' + name.lower() + '}'
def _macro_split(self, s):
'''Split the string (s) up by macros. Only split on the
outter level. Nested levels will need to split with futher calls.'''
trace_me = False
if trace_me:
print('------------------------------------------------------')
macros = []
nesting = []
has_braces = False
c = 0
while c < len(s):
if trace_me:
print('ms:', c, '"' + s[c:] + '"', has_braces, len(nesting), nesting)
#
# We need to watch for shell type variables or the form '${var}' because
# they can upset the brace matching.
#
if s[c] == '%' or s[c] == '$':
start = s[c]
c += 1
if c == len(s):
continue
#
# Do we have '%%' or '%(' or '$%' or '$(' or not '${' ?
#
if s[c] == '%' or s[c] == '(' or (start == '$' and s[c] != '{'):
continue
elif not s[c].isspace():
#
# If this is a shell macro and we are at the outter
# level or is '$var' forget it and move on.
#
if start == '$' and (s[c] != '{' or len(nesting) == 0):
continue
if s[c] == '{':
this_has_braces = True
else:
this_has_braces = False
nesting.append((c - 1, has_braces))
has_braces = this_has_braces
elif len(nesting) > 0:
if s[c] == '}' or (s[c].isspace() and not has_braces):
#
# Can have '%{?test: something %more}' where the
# nested %more ends with the '}' which also ends
# the outter macro.
#
if not has_braces:
if s[c] == '}':
macro_start, has_braces = nesting[len(nesting) - 1]
nesting = nesting[:-1]
if len(nesting) == 0:
macros.append(s[macro_start:c].strip())
if len(nesting) > 0:
macro_start, has_braces = nesting[len(nesting) - 1]
nesting = nesting[:-1]
if len(nesting) == 0:
macros.append(s[macro_start:c + 1].strip())
c += 1
if trace_me:
print('ms:', macros)
if trace_me:
print('-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=')
return macros
def _shell(self, line):
sl = self.sf.findall(line)
if len(sl):
e = execute.capture_execution()
for s in sl:
if host.is_windows:
cmd = '%s -c "%s"' % (self.macros.expand('%{__sh}'), s[2:-1])
else:
cmd = s[2:-1]
exit_code, proc, output = e.shell(cmd)
if exit_code == 0:
line = line.replace(s, output)
else:
raise error.general('shell macro failed: %s:%d: %s' % (s, exit_code, output))
return line
def _expand(self, s):
expand_count = 0
expanded = True
while expanded:
expand_count += 1
if expand_count > 500:
raise error.general('macro expand looping: %s' % (s))
expanded = False
ms = self._macro_split(s)
for m in ms:
mn = m
#
# A macro can be '%{macro}' or '%macro'. Turn the later into
# the former.
#
show_warning = True
if mn[1] != '{':
if self.ignores is not None:
for r in self.ignores:
if r.match(mn) is not None:
mn = None
break
else:
mn = self._label(mn[1:])
show_warning = False
else:
mn = self._label(mn[1:])
show_warning = False
elif m.startswith('%{expand'):
colon = m.find(':')
if colon < 8:
log.warning('malformed expand macro, no colon found')
else:
e = self._expand(m[colon + 1:-1].strip())
s = s.replace(m, e)
expanded = True
mn = None
elif m.startswith('%{with '):
#
# Change the ' ' to '_' because the macros have no spaces.
#
n = self._label('with_' + m[7:-1].strip())
if n in self.macros:
s = s.replace(m, '1')
else:
s = s.replace(m, '0')
expanded = True
mn = None
elif m.startswith('%{echo'):
if not m.endswith('}'):
log.warning("malformed conditional macro '%s'" % (m))
mn = None
else:
e = self._expand(m[6:-1].strip())
log.output('%s' % (self._name_line_msg(e)))
s = ''
expanded = True
mn = None
elif m.startswith('%{defined'):
n = self._label(m[9:-1].strip())
if n in self.macros:
s = s.replace(m, '1')
else:
s = s.replace(m, '0')
expanded = True
mn = None
elif m.startswith('%{?') or m.startswith('%{!?'):
if m[2] == '!':
start = 4
else:
start = 3
colon = m[start:].find(':')
if colon < 0:
if not m.endswith('}'):
log.warning("malformed conditional macro '%s'" % (m))
mn = None
else:
mn = self._label(m[start:-1])
else:
mn = self._label(m[start:start + colon])
if mn:
if m.startswith('%{?'):
istrue = False
if mn in self.macros:
# If defined and 0 then it is false.
istrue = _check_bool(self.macros[mn])
if istrue is None:
istrue = True
if colon >= 0 and istrue:
s = s.replace(m, m[start + colon + 1:-1])
expanded = True
mn = None
elif not istrue:
mn = '%{nil}'
else:
isfalse = True
if mn in self.macros:
istrue = _check_bool(self.macros[mn])
if istrue is None or istrue == True:
isfalse = False
if colon >= 0 and isfalse:
s = s.replace(m, m[start + colon + 1:-1])
expanded = True
mn = None
else:
mn = '%{nil}'
if mn:
if mn.lower() in self.macros:
s = s.replace(m, self.macros[mn.lower()])
expanded = True
elif show_warning:
self._error("macro '%s' not found" % (mn))
return self._shell(s)
def _disable(self, config, ls):
if len(ls) != 2:
log.warning('invalid disable statement')
else:
if ls[1] == 'select':
self.macros.lock_read_map()
log.trace('config: %s: _disable_select: %s' % (self.init_name, ls[1]))
else:
log.warning('invalid disable statement: %s' % (ls[1]))
def _select(self, config, ls):
if len(ls) != 2:
log.warning('invalid select statement')
else:
r = self.macros.set_read_map(ls[1])
log.trace('config: %s: _select: %s %s %r' % \
(self.init_name, r, ls[1], self.macros.maps()))
def _define(self, config, ls):
if len(ls) <= 1:
log.warning('invalid macro definition')
else:
d = self._label(ls[1])
if self.disable_macro_reassign:
if (d not in self.macros) or \
(d in self.macros and len(self.macros[d]) == 0):
if len(ls) == 2:
self.macros[d] = '1'
else:
self.macros[d] = ' '.join([f.strip() for f in ls[2:]])
else:
log.warning("macro '%s' already defined" % (d))
else:
if len(ls) == 2:
self.macros[d] = '1'
else:
self.macros[d] = ' '.join([f.strip() for f in ls[2:]])
def _undefine(self, config, ls):
if len(ls) <= 1:
log.warning('invalid macro definition')
else:
mn = self._label(ls[1])
if mn in self.macros:
del self.macros[mn]
else:
log.warning("macro '%s' not defined" % (mn))
def _ifs(self, config, ls, label, iftrue, isvalid, dir, info):
in_iftrue = True
data = []
while True:
if isvalid and \
((iftrue and in_iftrue) or (not iftrue and not in_iftrue)):
this_isvalid = True
else:
this_isvalid = False
r = self._parse(config, dir, info, roc = True, isvalid = this_isvalid)
if r[0] == 'control':
if r[1] == '%end':
self._error(label + ' without %endif')
raise error.general('terminating build')
if r[1] == '%endif':
log.trace('config: %s: _ifs: %s %s' % (self.init_name, r[1], this_isvalid))
return data
if r[1] == '%else':
in_iftrue = False
elif r[0] == 'directive':
if this_isvalid:
if r[1] == '%include':
self.load(r[2][0])
continue
dir, info, data = self._process_directive(r, dir, info, data)
elif r[0] == 'data':
if this_isvalid:
dir, info, data = self._process_data(r, dir, info, data)
else:
dir, info, data = self._process_block(r, dir, info, data)
# @note is a directive extend missing
def _if(self, config, ls, isvalid, dir, info, invert = False):
def add(x, y):
return x + ' ' + str(y)
istrue = False
if isvalid:
if len(ls) == 2:
s = ls[1]
else:
s = (ls[1] + ' ' + ls[2])
ifls = s.split()
if len(ifls) == 1:
#
# Check if '%if %{x} == %{nil}' has both parts as nothing
# which means '%if ==' is always True and '%if !=' is always false.
#
if ifls[0] == '==':
istrue = True
elif ifls[0] == '!=':
istrue = False
else:
istrue = _check_bool(ifls[0])
if istrue == None:
self._error('invalid if bool value: ' + functools.reduce(add, ls, ''))
istrue = False
elif len(ifls) == 2:
if ifls[0] == '!':
istrue = _check_bool(ifls[1])
if istrue == None:
self._error('invalid if bool value: ' + functools.reduce(add, ls, ''))
istrue = False
else:
istrue = not istrue
else:
#
# Check is something is being checked against empty,
# ie '%if %{x} == %{nil}'
# The logic is 'something == nothing' is False and
# 'something != nothing' is True.
#
if ifls[1] == '==':
istrue = False
elif ifls[1] == '!=':
istrue = True
else:
self._error('invalid if bool operator: ' + functools.reduce(add, ls, ''))
elif len(ifls) == 3:
if ifls[1] == '==':
if ifls[0] == ifls[2]:
istrue = True
else:
istrue = False
elif ifls[1] == '!=' or ifls[1] == '=!':
if ifls[0] != ifls[2]:
istrue = True
else:
istrue = False
elif ifls[1] == '>':
if ifls[0] > ifls[2]:
istrue = True
else:
istrue = False
elif ifls[1] == '>=' or ifls[1] == '=>':
if ifls[0] >= ifls[2]:
istrue = True
else:
istrue = False
elif ifls[1] == '<=' or ifls[1] == '=<':
if ifls[0] <= ifls[2]:
istrue = True
else:
istrue = False
elif ifls[1] == '<':
if ifls[0] < ifls[2]:
istrue = True
else:
istrue = False
else:
self._error('invalid %if operator: ' + functools.reduce(add, ls, ''))
else:
self._error('malformed if: ' + functools.reduce(add, ls, ''))
if invert:
istrue = not istrue
log.trace('config: %s: _if: %s %s' % (self.init_name, ifls, str(istrue)))
return self._ifs(config, ls, '%if', istrue, isvalid, dir, info)
def _ifos(self, config, ls, isvalid, dir, info):
isos = False
if isvalid:
os = self.define('_os')
for l in ls:
if l in os:
isos = True
break
return self._ifs(config, ls, '%ifos', isos, isvalid, dir, info)
def _ifarch(self, config, positive, ls, isvalid, dir, info):
isarch = False
if isvalid:
arch = self.define('_arch')
for l in ls:
if l in arch:
isarch = True
break
if not positive:
isarch = not isarch
return self._ifs(config, ls, '%ifarch', isarch, isvalid, dir, info)
def _parse(self, config, dir, info, roc = False, isvalid = True):
# roc = return on control
def _clean(line):
line = line[0:-1]
b = line.find('#')
if b >= 0:
line = line[1:b]
return line.strip()
#
# Need to add code to count matching '{' and '}' and if they
# do not match get the next line and add to the string until
# they match. This closes an opening '{' that is on another
# line.
#
for l in config:
self.lc += 1
l = _clean(l)
if len(l) == 0:
continue
log.trace('config: %s: %03d: %s %s' % \
(self.init_name, self.lc, str(isvalid), l))
lo = l
if isvalid:
l = self._expand(l)
if len(l) == 0:
continue
if l[0] == '%':
ls = self.wss.split(l, 2)
los = self.wss.split(lo, 2)
if ls[0] == '%disable':
if isvalid:
self._disable(config, ls)
elif ls[0] == '%select':
if isvalid:
self._select(config, ls)
elif ls[0] == '%error':
if isvalid:
return ('data', ['%%error %s' % (self._name_line_msg(l[7:]))])
elif ls[0] == '%warning':
if isvalid:
return ('data', ['%%warning %s' % (self._name_line_msg(l[9:]))])
elif ls[0] == '%define' or ls[0] == '%global':
if isvalid:
self._define(config, ls)
elif ls[0] == '%undefine':
if isvalid:
self._undefine(config, ls)
elif ls[0] == '%if':
d = self._if(config, ls, isvalid, dir, info)
if len(d):
log.trace('config: %s: %%if: %s' % (self.init_name, d))
return ('data', d)
elif ls[0] == '%ifn':
d = self._if(config, ls, isvalid, dir, info, True)
if len(d):
log.trace('config: %s: %%ifn: %s' % (self.init_name, d))
return ('data', d)
elif ls[0] == '%ifos':
d = self._ifos(config, ls, isvalid, dir, info)
if len(d):
return ('data', d)
elif ls[0] == '%ifarch':
d = self._ifarch(config, True, ls, isvalid, dir, info)
if len(d):
return ('data', d)
elif ls[0] == '%ifnarch':
d = self._ifarch(config, False, ls, isvalid, dir, info)
if len(d):
return ('data', d)
elif ls[0] == '%endif':
if roc:
return ('control', '%endif', '%endif')
log.warning("unexpected '" + ls[0] + "'")
elif ls[0] == '%else':
if roc:
return ('control', '%else', '%else')
log.warning("unexpected '" + ls[0] + "'")
elif ls[0].startswith('%defattr'):
return ('data', [l])
elif ls[0] == '%bcond_with':
if isvalid:
#
# Check if already defined. Would be by the command line or
# even a host specific default.
#
if self._label('with_' + ls[1]) not in self.macros:
self._define(config, (ls[0], 'without_' + ls[1]))
elif ls[0] == '%bcond_without':
if isvalid:
if self._label('without_' + ls[1]) not in self.macros:
self._define(config, (ls[0], 'with_' + ls[1]))
else:
pt = self._parse_token(lo, los, l, ls)
if pt is not None:
return pt
if self.ignores is not None:
for r in self.ignores:
if r.match(ls[0]) is not None:
return ('data', [l])
if isvalid:
for d in self.directives:
if ls[0].strip() == d:
return ('directive', ls[0].strip(), ls[1:])
log.warning("unknown directive: '" + ls[0] + "'")
return ('data', [lo])
else:
return ('data', [lo])
return ('control', '%end', '%end')
def _parse_token(self, line, line_split, line_expanded, line_split_expanded):
return None
def _process_directive(self, results, directive, info, data):
new_data = []
if results[1] == '%description':
new_data = [' '.join(results[2])]
else:
directive, into, data = self._directive_filter(results, directive, info, data)
if directive and directive != results[1]:
self._directive_extend(directive, data)
directive = results[1]
data = new_data
return (directive, info, data)
def _process_data(self, results, directive, info, data):
new_data = []
for l in results[1]:
if l.startswith('%error'):
l = self._expand(l)
raise error.general('config error: %s' % (l[7:]))
elif l.startswith('%warning'):
l = self._expand(l)
log.stderr('warning: %s' % (l[9:]))
log.warning(l[9:])
if not directive:
l = self._expand(l)
ls = self.tags.split(l, 1)
log.trace('config: %s: _tag: %s %s' % (self.init_name, l, ls))
if len(ls) > 1:
info = ls[0].lower()
if info[-1] == ':':
info = info[:-1]
info_data = ls[1].strip()
else:
info_data = ls[0].strip()
if info is not None:
self._info_append(info, info_data)
else:
log.warning("invalid format: '%s'" % (info_data[:-1]))
else:
log.trace('config: %s: _data: %s %s' % (self.init_name, l, new_data))
new_data.append(l)
return (directive, info, data + new_data)
def _process_block(self, results, directive, info, data):
raise error.internal('known block type: %s' % (results[0]))
def _directive_extend(self, dir, data):
pass
def _directive_filter(self, results, directive, info, data):
return directive, into, data
def _info_append(self, info, data):
pass
def load(self, name):
def common_end(left, right):
end = ''
while len(left) and len(right):
if left[-1] != right[-1]:
return end
end = left[-1] + end
left = left[:-1]
right = right[:-1]
return end
if self.load_depth == 0:
self.in_error = False
self.lc = 0
self.name = name
self.conditionals = {}
self.load_depth += 1
save_name = self.name
save_lc = self.lc
self.name = name
self.lc = 0
#
# Locate the config file. Expand any macros then add the
# extension. Check if the file exists, therefore directly
# referenced. If not see if the file contains ':' or the path
# separator. If it does split the path else use the standard config dir
# path in the defaults.
#
exname = self.expand(name)
#
# Macro could add an extension.
#
if exname.endswith('.cfg'):
configname = exname
else:
configname = '%s.cfg' % (exname)
name = '%s.cfg' % (name)
if ':' in configname:
cfgname = path.basename(configname)
else:
cfgname = common_end(configname, name)
if not path.exists(configname):
if ':' in configname:
configdirs = path.dirname(configname).split(':')
else:
configdirs = self.define('_configdir').split(':')
for cp in configdirs:
configname = path.join(path.abspath(cp), cfgname)
if path.exists(configname):
break
configname = None
if configname is None:
raise error.general('no config file found: %s' % (cfgname))
try:
log.trace('config: %s: _open: %s' % (self.init_name, path.host(configname)))
config = open(path.host(configname), 'r')
except IOError as err:
raise error.general('error opening config file: %s' % (path.host(configname)))
self.configpath += [configname]
self._includes += [configname]
try:
dir = None
info = None
data = []
while True:
r = self._parse(config, dir, info)
if r[0] == 'control':
if r[1] == '%end':
break
log.warning("unexpected '%s'" % (r[1]))
elif r[0] == 'directive':
if r[1] == '%include':
self.load(r[2][0])
continue
dir, info, data = self._process_directive(r, dir, info, data)
elif r[0] == 'data':
dir, info, data = self._process_data(r, dir, info, data)
else:
self._error("%d: invalid parse state: '%s" % (self.lc, r[0]))
if dir is not None:
self._directive_extend(dir, data)
except:
config.close()
raise
config.close()
self.name = save_name
self.lc = save_lc
self.load_depth -= 1
def defined(self, name):
return self.macros.has_key(name)
def define(self, name):
if name in self.macros:
d = self.macros[name]
else:
n = self._label(name)
if n in self.macros:
d = self.macros[n]
else:
raise error.general('%d: macro "%s" not found' % (self.lc, name))
return self._expand(d)
def set_define(self, name, value):
self.macros[name] = value
def expand(self, line):
if type(line) == list:
el = []
for l in line:
el += [self._expand(l)]
return el
return self._expand(line)
def macro(self, name):
if name in self.macros:
return self.macros[name]
raise error.general('macro "%s" not found' % (name))
def directive(self, name):
pass
def abspath(self, rpath):
return path.abspath(self.define(rpath))
def includes(self):
return self._includes
def file_name(self):
return self.init_name
def run():
import sys
try:
#
# Run where defaults.mc is located
#
long_opts = {
# key macro handler param defs init
'--file' : ('_file', 'path', True, None, False)
}
opts = options.command_line(base_path = '.',
argv = sys.argv,
long_opts = long_opts)
options.load(opts)
s = file(opts.defaults['_file'], opts)
s.load(opts.defaults['_file'])
print(s)
del s
except error.general as gerr:
print(gerr)
sys.exit(1)
except error.internal as ierr:
print(ierr)
sys.exit(1)
except KeyboardInterrupt:
log.notice('abort: user terminated')
sys.exit(1)
sys.exit(0)
if __name__ == "__main__":
run()