Add macro include support. Use it for building from head.

This change provides a simple way to build all parts of the tools from
version control.

doc/source-builder.txt

@@ -779,16 +779,17 @@ and the 'String' field is a single or tripled multiline quoted string. The
 'String' can contain references to other macros. Macro that loop are not
 currently detected and will cause the tool to lock up.
 
-Maps are declared anywhere in the map using:
+Maps are declared anywhere in the map using the map directive:
 
 -------------------------------------------------------------
 # Comments
-[my-special-map]
+[my-special-map] <1>
 _host:  none, override, 'abc-xyz'
 multiline: none, override, '''First line,
 second line,
 and finally the last line'''
 -------------------------------------------------------------
+<1> The map is set to `my-special-map`.
 
 Any macro defintions following a map declaration are placed in that map and the
 default map is `global` when loading a file. Maps are selected in configuration
@@ -803,8 +804,27 @@ if present return that value else the `global` map is used. Any new macros or
 changes update only the `global` map. This may change in future releases so
 please make sure you use the `override` attibute.
 
-The macro files are looked for in the `_configdir` paths. See
+The macro files specificed on the command line are looked for in the
-<<X1,+_configdir+>> variable for details.
+`_configdir` paths. See <<X1,+_configdir+>> variable for details. Included
+files need to add the `%{_configdir}` macro to the start of the file.
+
+Macro map files can include other macro map files using the `%include`
+directive. The macro map to build _binutils_, _gcc_, _newlib_, _gdb_ and
+_RTEMS_ from version control heads is:
+
+-------------------------------------------------------------
+# <1>
+# Build all tool parts from version control head.
+#
+%include %{_configdir}/snapshots/binutils-head.mc
+%include %{_configdir}/snapshots/gcc-head.mc
+%include %{_configdir}/snapshots/newlib-head.mc
+%include %{_configdir}/snapshots/gdb-head.mc
+-------------------------------------------------------------
+<1> The file is `config/snapshots/binutils-gcc-newlib-gdb-head.mc`.
+
+The macro map defaults to `global` at the start of each included file and the
+map setting of the macro file including the other macro files does not change.
 
 Build Set Files
 ~~~~~~~~~~~~~~~
@@ -1638,12 +1658,23 @@ build fails a check.
 
 To build snapshots for testing purposes you use the available macro maps
 passing them on the command line using the `--macros` option. For RTEMS these
-are held in `config/snapshots` directory. The following build _newlib_ from
+are held in `config/snapshots` directory. The following builds _newlib_ from
 CVS:
 
 -------------------------------------------------------------
 $ ../source-builder/sb-set-builder --log=l-4.11-sparc.txt \
-   --prefix=$HOME/development/rtems/4.11 --macros=snapshots/newlib-head.mc
+   --prefix=$HOME/development/rtems/4.11 \
+   --macros=snapshots/newlib-head.mc \
+   4.11/rtems-sparc
+-------------------------------------------------------------
+
+and the following uses the version control heads for _binutils_, _gcc_,
+_newlib_, _gdb_ and _RTEMS_:
+
+-------------------------------------------------------------
+$ ../source-builder/sb-set-builder --log=l-heads-sparc.txt \
+   --prefix=$HOME/development/rtems/4.11-head \
+   --macros=snapshots/binutils-gcc-newlib-gdb-head.mc \
    4.11/rtems-sparc
 -------------------------------------------------------------
 

source-builder/config/snapshots/binutils-gcc-newlib-gdb-head.mc

@@ -0,0 +1,7 @@
+#
+# Build all tool parts from version control head.
+#
+%include %{_configdir}/snapshots/binutils-head.mc
+%include %{_configdir}/snapshots/gcc-head.mc
+%include %{_configdir}/snapshots/newlib-head.mc
+%include %{_configdir}/snapshots/gdb-head.mc

source-builder/config/snapshots/binutils-head.mc

@@ -0,0 +1,13 @@
+[binutils-snapshot]
+Binutils_Version: none, override, 'cvs-head'
+Source0:          none, override, 'cvs://pserver:anoncvs@sourceware.org/cvs/src?module=binutils?src-prefix=src?update'
+Patch0:           none, undefine, ''
+Patch1:           none, undefine, ''
+Patch2:           none, undefine, ''
+Patch3:           none, undefine, ''
+Patch4:           none, undefine, ''
+Patch5:           none, undefine, ''
+Patch6:           none, undefine, ''
+Patch7:           none, undefine, ''
+Patch8:           none, undefine, ''
+Patch9:           none, undefine, ''

source-builder/config/snapshots/gcc-head.mc

@@ -0,0 +1,13 @@
+[gcc-snapshot]
+GCC_Version: none, override, 'git-head'
+Source0:     none, override, 'git://gcc.gnu.org/git/gcc.git?branch=master?pull'
+Patch0:      none, undefine, ''
+Patch1:      none, undefine, ''
+Patch2:      none, undefine, ''
+Patch3:      none, undefine, ''
+Patch4:      none, undefine, ''
+Patch5:      none, undefine, ''
+Patch6:      none, undefine, ''
+Patch7:      none, undefine, ''
+Patch8:      none, undefine, ''
+Patch9:      none, undefine, ''

source-builder/config/snapshots/gdb-head.mc

@@ -1,6 +1,6 @@
 [gdb-snapshot]
 GDB_Version: none, override, 'cvs-head'
-Source0:     none, override, 'cvs://pserver:anoncvs@sourceware.org/cvs/src?module=gdb?src-prefix=src'
+Source0:     none, override, 'cvs://pserver:anoncvs@sourceware.org/cvs/src?module=gdb?src-prefix=src?update'
 Patch0:      none, undefine, ''
 Patch1:      none, undefine, ''
 Patch2:      none, undefine, ''

source-builder/sb/macros.py

@@ -186,6 +186,19 @@ class macros:
         return key.lower()
 
     def parse(self, lines):
+
+        def _clean(l):
+            if '#' in l:
+                l = l[:l.index('#')]
+            if '\r' in l:
+                l = l[:l.index('r')]
+            if '\n' in l:
+                l = l[:l.index('\n')]
+            return l.strip()
+
+        trace_me = False
+        if trace_me:
+            print '[[[[]]]] parsing macros'
         macros = { 'global': {} }
         map = 'global'
         lc = 0
@@ -197,8 +210,12 @@ class macros:
             #print 'l:%s' % (l[:-1])
             if len(l) == 0:
                 continue
+            l_remaining = l
             for c in l:
-                #print ']]]]]]]] c:%s(%d) s:%s t:"%s" m:%r M:%s' % (c, ord(c), state, token, macro, map)
+                if trace_me:
+                    print ']]]]]]]] c:%s(%d) s:%s t:"%s" m:%r M:%s' % \
+                        (c, ord(c), state, token, macro, map)
+                l_remaining = l_remaining[1:]
                 if c is '#' and not state.startswith('value'):
                     break
                 if c == '\n' or c == '\r':
@@ -209,6 +226,8 @@ class macros:
                     if c not in string.whitespace:
                         if c is '[':
                             state = 'map'
+                        elif c is '%':
+                            state = 'directive'
                         elif c is ':':
                             macro += [token]
                             token = ''
@@ -228,6 +247,25 @@ class macros:
                         token += c
                     else:
                         raise error.general('invalid macro map:%d: %s' % (lc, l))
+                elif state is 'directive':
+                    if c in string.whitespace:
+                        if token == 'include':
+                            self.load(_clean(l_remaining))
+                            token = ''
+                            state = 'key'
+                            break
+                    elif c in string.printable and c not in string.whitespace:
+                        token += c
+                    else:
+                        raise error.general('invalid macro directive:%d: %s' % (lc, l))
+                elif state is 'include':
+                    if c is string.whitespace:
+                        if token == 'include':
+                            state = 'include'
+                    elif c in string.printable and c not in string.whitespace:
+                        token += c
+                    else:
+                        raise error.general('invalid macro directive:%d: %s' % (lc, l))
                 elif state is 'attribs':
                     if c not in string.whitespace:
                         if c is ',':
@@ -290,14 +328,18 @@ class macros:
                 self.macros[m][mm] = macros[m][mm]
 
     def load(self, name):
-        try:
+        names = self.expand(name).split(':')
-            name = self.expand(name)
+        for n in names:
-            mc = open(name, 'r')
+            if path.exists(n):
-        except IOError, err:
+                try:
-            raise error.general('opening macro file: %s' % (path.host(name)))
+                    mc = open(n, 'r')
-        macros = self.parse(mc)
+                    macros = self.parse(mc)
-        mc.close()
+                    mc.close()
-        self.files += [name]
+                    self.files += [n]
+                    return
+                except IOError, err:
+                    pass
+        raise error.general('opening macro file: %s' % (path.host(name)))
 
     def get(self, key):
         if type(key) is not str: