sb: Implement %source and %patch to manage source and patches.

Remove the numbered source and patches and automatically manage
sources and patches. This removes the overhead in maintaining large
collections of patches.

source-builder/sb/build.py

@@ -41,6 +41,7 @@ try:
     import log
     import options
     import path
+    import sources
     import version
 except KeyboardInterrupt:
     print 'abort: user terminated'
@@ -131,63 +132,6 @@ class build:
         if not self.opts.dry_run():
             path.mkdir(mkpath)
 
-    def source(self, package, source_tag):
-        #
-        # Scan the sources found in the config file for the one we are
-        # after. Infos or tags are lists. Merge in any macro defined
-        # sources as these may be overridden by user loaded macros.
-        #
-        sources = package.sources()
-        url = None
-        for s in sources:
-            tag = s[len('source'):]
-            if tag.isdigit():
-                if int(tag) == source_tag:
-                    url = sources[s][0]
-                    break
-        if url is None:
-            raise error.general('source tag not found: source%d' % (source_tag))
-        source = download.parse_url(url, '_sourcedir', self.config, self.opts)
-        download.get_file(source['url'], source['local'], self.opts, self.config)
-        if 'symlink' in source:
-            source['script'] = '%%{__ln_s} %s ${source_dir_%d}' % (source['symlink'], source_tag)
-        elif 'compressed' in source:
-            source['script'] = source['compressed'] + ' ' + \
-                source['local'] + ' | %{__tar_extract} -'
-        else:
-            source['script'] = '%{__tar_extract} ' + source['local']
-        return source
-
-    def patch(self, package, args):
-        #
-        # Scan the patches found in the config file for the one we are
-        # after. Infos or tags are lists.
-        #
-        patches = package.patches()
-        url = None
-        for p in patches:
-            if args[0][1:].lower() == p:
-                url = patches[p][0]
-                break
-        if url is None:
-            raise error.general('patch tag not found: %s' % (args[0]))
-        #
-        # Parse the URL first in the source builder's patch directory.
-        #
-        patch = download.parse_url(url, '_patchdir', self.config, self.opts)
-        #
-        # If not in the source builder package check the source directory.
-        #
-        if not path.exists(patch['local']):
-            patch = download.parse_url(url, '_patchdir', self.config, self.opts)
-        download.get_file(patch['url'], patch['local'], self.opts, self.config)
-        if 'compressed' in patch:
-            patch['script'] = patch['compressed'] + ' ' +  patch['local']
-        else:
-            patch['script'] = '%{__cat} ' + patch['local']
-        patch['script'] += ' | %{__patch} ' + ' '.join(args[1:])
-        self.script.append(self.config.expand(patch['script']))
-
     def canadian_cross(self):
         _host = self.config.expand('%{_host}')
         _build = self.config.expand('%{_build}')
@@ -195,16 +139,54 @@ class build:
         return self.config.defined('%{allow_cxc}') and \
             _host != _build and _host != _target
 
-    def setup(self, package, args):
-        log.output('prep: %s: %s' % (package.name(), ' '.join(args)))
-        opts, args = getopt.getopt(args[1:], 'qDcTn:b:a:')
-        source_tag = 0
+    def source(self, name):
+        #
+        # Return the list of sources. Merge in any macro defined sources as
+        # these may be overridden by user loaded macros.
+        #
+        _map = 'source-%s' % (name)
+        src_keys = self.macros.map_keys(_map)
+        if len(src_keys) == 0:
+            raise error.general('no source set: %s (%s)' % (name, _map))
+        srcs = []
+        for s in src_keys:
+            sm = self.macros.get(s, globals = False, maps = _map)
+            if sm is None:
+                raise error.internal('source macro not found: %s in %s (%s)' % \
+                                         (s, name, _map))
+            url = self.config.expand(sm[2])
+            src = download.parse_url(url, '_sourcedir', self.config, self.opts)
+            download.get_file(src['url'], src['local'], self.opts, self.config)
+            if 'symlink' in src:
+                src['script'] = '%%{__ln_s} %s ${source_dir_%s}' % (src['symlink'], name)
+            elif 'compressed' in src:
+                #
+                # Zip files unpack as well so do not use tar.
+                #
+                src['script'] = '%s %s' % (src['compressed'], src['local'])
+                if src['compressed-type'] != 'zip':
+                    src['script'] += ' | %{__tar_extract} -'
+            else:
+                src['script'] = '%{__tar_extract} %s' % (src['local'])
+            srcs += [src]
+        return srcs
+
+    def source_setup(self, package, args):
+        log.output('source setup: %s: %s' % (package.name(), ' '.join(args)))
+        setup_name = args[1]
+        args = args[1:]
+        try:
+            opts, args = getopt.getopt(args[1:], 'qDcn:b:a:')
+        except getopt.GetoptError, ge:
+            raise error.general('source setup error: %s' % str(ge))
         quiet = False
-        unpack_default_source = True
         unpack_before_chdir = True
         delete_before_unpack = True
         create_dir = False
-        name = None
+        deleted_dir = False
+        created_dir = False
+        changed_dir = False
+        opt_name = None
         for o in opts:
             if o[0] == '-q':
                 quiet = True
@@ -212,51 +194,83 @@ class build:
                 delete_before_unpack = False
             elif o[0] == '-c':
                 create_dir = True
-            elif o[0] == '-T':
-                unpack_default_source = False
             elif o[0] == '-n':
-                name = o[1]
+                opt_name = o[1]
             elif o[0] == '-b':
                 unpack_before_chdir = True
-                if not o[1].isdigit():
-                    raise error.general('setup -b source tag is not a number: %s' % (o[1]))
-                source_tag = int(o[1])
             elif o[0] == '-a':
                 unpack_before_chdir = False
-                if not o[1].isdigit():
-                    raise error.general('setup -a source tag is not a number: %s' % (o[1]))
-                source_tag = int(o[1])
-        source0 = None
-        source = self.source(package, source_tag)
-        if name is None:
-            if source:
-                name = source['name']
-            else:
-                raise error.general('setup source tag not found: %d' % (source_tag))
-        name = self._name_(name)
-        self.script.append(self.config.expand('cd %{_builddir}'))
-        if delete_before_unpack:
-            self.script.append(self.config.expand('%{__rm} -rf ' + name))
-        if create_dir:
-            self.script.append(self.config.expand('%{__mkdir_p} ' + name))
-        #
-        # If -a? then change directory before unpacking.
-        #
-        if not unpack_before_chdir or create_dir:
-            self.script.append(self.config.expand('cd ' + name))
-        #
-        # Unpacking the source. Note, treated the same as -a0.
-        #
-        if unpack_default_source and source_tag != 0:
-            source0 = self.source(package, 0)
-            if source0 is None:
-                raise error.general('no setup source0 tag found')
-            self.script.append(self.config.expand(source0['script']))
-        self.script.append(self.config.expand(source['script']))
-        if unpack_before_chdir and not create_dir:
+        name = None
+        for source in self.source(setup_name):
+            if name is None:
+                if opt_name is None:
+                    if source:
+                        opt_name = source['name']
+                    else:
+                        raise error.general('setup source tag not found: %d' % (source_tag))
+                else:
+                    name = opt_name
+                name = self._name_(name)
+            self.script.append(self.config.expand('cd %{_builddir}'))
+            if not deleted_dir and  delete_before_unpack:
+                self.script.append(self.config.expand('%{__rm} -rf ' + name))
+                deleted_dir = True
+            if not created_dir and create_dir:
+                self.script.append(self.config.expand('%{__mkdir_p} ' + name))
+                created_dir = True
+            if not changed_dir and (not unpack_before_chdir or create_dir):
+                self.script.append(self.config.expand('cd ' + name))
+                changed_dir = True
+            self.script.append(self.config.expand(source['script']))
+        if not changed_dir and (unpack_before_chdir and not create_dir):
             self.script.append(self.config.expand('cd ' + name))
+            changed_dir = True
         self.script.append(self.config.expand('%{__setup_post}'))
 
+    def patch_setup(self, package, args):
+        name = args[1]
+        args = args[2:]
+        _map = 'patch-%s' % (name)
+        default_opts = ' '.join(args)
+        patch_keys = self.macros.map_keys(_map)
+        patches = []
+        for p in patch_keys:
+            pm = self.macros.get(p, globals = False, maps = _map)
+            if pm is None:
+                raise error.internal('patch macro not found: %s in %s (%s)' % \
+                                         (p, name, _map))
+            opts = []
+            url = []
+            for pp in pm[2].split():
+                if len(url) == 0 and pp[0] == '-':
+                    opts += [pp]
+                else:
+                    url += [pp]
+            if len(url) == 0:
+                raise error.general('patch URL not found: %s' % (' '.join(args)))
+            if len(opts) == 0:
+                opts = default_opts
+            else:
+                opts = ' '.join(opts)
+            opts = self.config.expand(opts)
+            url = self.config.expand(' '.join(url))
+            #
+            # Parse the URL first in the source builder's patch directory.
+            #
+            patch = download.parse_url(url, '_patchdir', self.config, self.opts)
+            #
+            # If not in the source builder package check the source directory.
+            #
+            if not path.exists(patch['local']):
+                patch = download.parse_url(url, '_patchdir', self.config, self.opts)
+            download.get_file(patch['url'], patch['local'], self.opts, self.config)
+            if 'compressed' in patch:
+                patch['script'] = patch['compressed'] + ' ' +  patch['local']
+            else:
+                patch['script'] = '%{__cat} ' + patch['local']
+            patch['script'] += ' | %%{__patch} %s' % (opts)
+            self.script.append(self.config.expand(patch['script']))
+
     def run(self, command, shell_opts = '', cwd = None):
         e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
         cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
@@ -278,12 +292,18 @@ class build:
         if _prep:
             for l in _prep:
                 args = l.split()
-                if args[0] == '%setup':
-                    self.setup(package, args)
-                elif args[0].startswith('%patch'):
-                    self.patch(package, args)
-                else:
-                    self.script.append(' '.join(args))
+                if len(args):
+                    if args[0] == '%setup':
+                        if len(args) == 1:
+                            raise error.general('invalid %%setup directive: %s' % (' '.join(args)))
+                        if args[1] == 'source':
+                            self.source_setup(package, args[1:])
+                        elif args[1] == 'patch':
+                            self.patch_setup(package, args[1:])
+                    elif args[0].startswith('%patch'):
+                        self.patch(package, args)
+                    else:
+                        self.script.append(' '.join(args))
 
     def build(self, package):
         self.script.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')

source-builder/sb/config.py

@@ -37,6 +37,7 @@ try:
     import options
     import path
     import pkgconfig
+    import sources
 except KeyboardInterrupt:
     print 'user terminated'
     sys.exit(1)
@@ -580,6 +581,9 @@ class file:
             log.trace('config: %s: _select: %s %s %r' % \
                           (self.init_name, r, ls[1], self.macros.maps()))
 
+    def _sources(self, ls):
+        return sources.process(ls[0][1:], ls[1:], self.macros, self._error)
+
     def _define(self, config, ls):
         if len(ls) <= 1:
             log.warning('invalid macro definition')
@@ -796,6 +800,14 @@ class file:
                 elif ls[0] == '%select':
                     if isvalid:
                         self._select(config, ls)
+                elif ls[0] == '%source' or ls[0] == '%patch':
+                    if isvalid:
+                        d = self._sources(ls)
+                        if d is not None:
+                            return ('data', d)
+                elif ls[0] == '%patch':
+                    if isvalid:
+                        self._select(config, ls)
                 elif ls[0] == '%error':
                     if isvalid:
                         return ('data', ['%%error %s' % (self._name_line_msg(l[7:]))])

source-builder/sb/download.py

@@ -40,12 +40,16 @@ def _http_parser(source, config, opts):
     #
     esl = source['ext'].split('.')
     if esl[-1:][0] == 'gz':
+        source['compressed-type'] = 'gzip'
         source['compressed'] = '%{__gzip} -dc'
     elif esl[-1:][0] == 'bz2':
+        source['compressed-type'] = 'bzip2'
         source['compressed'] = '%{__bzip2} -dc'
     elif esl[-1:][0] == 'zip':
+        source['compressed-type'] = 'zip'
         source['compressed'] = '%{__zip} -u'
     elif esl[-1:][0] == 'xz':
+        source['compressed-type'] = 'xz'
         source['compressed'] = '%{__xz} -dc'
 
 def _patchworks_parser(source, config, opts):
@@ -138,6 +142,9 @@ def parse_url(url, pathkey, config, opts):
     source['path'] = url[:colon + 3] + path.dirname(url[colon + 3:])
     source['file'] = path.basename(url)
     source['name'], source['ext'] = path.splitext(source['file'])
+    if source['name'].endswith('.tar'):
+        source['name'] = source['name'][:-4]
+        source['ext'] = '.tar' + source['ext']
     #
     # Get the file. Checks the local source directory first.
     #

source-builder/sb/macros.py

@@ -84,7 +84,10 @@ class macros:
         text = ''
         for f in self.files:
             text += '> %s%s' % (f, os.linesep)
-        for map in self.macros:
+        maps = sorted(self.macros)
+        maps.remove('global')
+        maps += ['global']
+        for map in maps:
             text += '[%s]%s' % (map, os.linesep)
             for k in sorted(self.macros[map].keys()):
                 d = self.macros[map][k]
@@ -162,8 +165,11 @@ class macros:
     def __len__(self):
         return len(self.keys())
 
-    def keys(self):
-        keys = self.macros['global'].keys()
+    def keys(self, globals = True):
+        if globals:
+            keys = self.macros['global'].keys()
+        else:
+            keys = []
         for rm in self.get_read_maps():
             for mk in self.macros[rm]:
                 if self.macros[rm][mk][1] == 'undefine':
@@ -180,9 +186,25 @@ class macros:
             return False
         return True
 
+    def create_map(self, _map):
+        if _map not in self.macros:
+            self.macros[_map] = {}
+
+    def delete_map(self, _map):
+        if _map in self.macros:
+            self.macros.pop(_map, None)
+
     def maps(self):
         return self.macros.keys()
 
+    def map_keys(self, _map):
+        if _map in self.macros:
+            return self.macros[_map].keys()
+        return []
+
+    def map_num_keys(self, _map):
+        return len(self.map_keys(_map))
+
     def get_read_maps(self):
         return [rm[5:] for rm in self.read_maps]
 
@@ -348,14 +370,21 @@ class macros:
         raise error.general('opening macro file: %s' % \
                                 (path.host(self.expand(name))))
 
-    def get(self, key):
+    def get(self, key, globals = True, maps = None):
         if type(key) is not str:
             raise TypeError('bad key type: %s' % (type(key)))
         key = self.key_filter(key)
-        for rm in self.get_read_maps():
+        if maps is None:
+            maps = self.get_read_maps()
+        else:
+            if type(maps) is str:
+                maps = [maps]
+            if type(maps) != list:
+                raise TypeError('bad maps type: %s' % (type(map)))
+        for rm in maps:
             if key in self.macros[rm]:
                 return self.macros[rm][key]
-        if key in self.macros['global']:
+        if globals and key in self.macros['global']:
             return self.macros['global'][key]
         return None
 
@@ -408,10 +437,10 @@ class macros:
                 expanded = True
         return _str
 
-    def find(self, regex):
+    def find(self, regex, globals = True):
         what = re.compile(regex)
         keys = []
-        for key in self.keys():
+        for key in self.keys(globals):
             if what.match(key):
                 keys += [key]
         return keys
@@ -440,6 +469,9 @@ class macros:
             return True
         return False
 
+    def unset_write_map(self):
+        self.write_map = 'global'
+
     def lock_read_map(self):
         self.read_map_locked = True
 

source-builder/sb/setbuilder.py

@@ -39,6 +39,7 @@ try:
     import options
     import path
     import reports
+    import sources
     import version
 except KeyboardInterrupt:
     print 'abort: user terminated'
@@ -248,9 +249,10 @@ class buildset:
                         self.macros.undefine(ls[1].strip())
                     elif ls[0] == '%include':
                         configs += self.parse(ls[1].strip())
-                    else:
-                        raise error.general('%s:%d: invalid directive in build set files: %s' % \
-                                                (self.bset, lc, l))
+                    elif ls[0] == '%patch' or ls[0] == '%source':
+                        def err(msg):
+                            raise error.general('%s:%d: %s' % (self.bset, lc, msg))
+                        sources.process(ls[0][1:], ls[1:], self.macros, err)
                 else:
                     l = l.strip()
                     c = build.find_config(l, self.configs)

source-builder/sb/sources.py

@@ -0,0 +1,74 @@
+#
+# RTEMS Tools Project (http://www.rtems.org/)
+# Copyright 2014 Chris Johns (chrisj@rtems.org)
+# All rights reserved.
+#
+# This file is part of the RTEMS Tools package in 'rtems-tools'.
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+#
+# Manage sources and patches
+#
+
+import log
+
+def _args(args):
+    return [i for s in [ii.split() for ii in args] for i in s]
+
+def add(label, args, macros, error):
+    args = _args(args)
+    if len(args) < 2:
+        error('%%%s requires at least 2 arguments' % (label))
+    _map = '%s-%s' % (label, args[0])
+    macros.create_map(_map)
+    macros.set_write_map(_map)
+    index = macros.map_num_keys(_map)
+    macros.define('%s%d' % (label, index), ' '.join(args[1:]))
+    macros.unset_write_map()
+    return None
+
+def set(label, args, macros, error):
+    args = _args(args)
+    if len(args) < 2:
+        error('%%%s requires at least 2 arguments' % (label))
+    _map = '%s-%s' % (label, args[0])
+    macros.create_map(_map)
+    key = '%s0' % (label)
+    if key not in macros.map_keys(_map):
+        macros.set_write_map(_map)
+        macros.define(key, ' '.join(args[1:]))
+        macros.unset_write_map()
+    return None
+
+def setup(label, args, macros, error):
+    args = _args(args)
+    if len(args) < 2:
+        error('%%%s requires at least 2 arguments: %s' % (label, ' '.join(args)))
+    _map = '%s-%s' % (label, args[0])
+    return ['%%setup %s %s' % (label, ' '.join(args))]
+
+def process(label, args, macros, error):
+    if label != 'source' and label != 'patch':
+        error('invalid source type: %s' % (label))
+    args = _args(args)
+    log.output('sources: %s' % (' '.join(args)))
+    if len(args) < 3:
+        error('%%%s requires at least 3 arguments: %s' % (label, ' '.join(args)))
+    if args[0] == 'set':
+        return set(label, args[1:], macros, error)
+    elif args[0] == 'add':
+        return add(label, args[1:], macros, error)
+    elif args[0] == 'setup':
+        return setup(label, args[1:], macros, error)
+    error('invalid %%%s command: %s' % (label, args[0]))

source-builder/sb/version.py

@@ -24,7 +24,7 @@
 #
 
 major = 0
-minor = 2
+minor = 3
 revision = 0
 
 def str():