sb: Backport from master the '--rsb-file=' option.

Upates #3033.

source-builder/sb/build.py

@@ -209,8 +209,33 @@ class build:
             if sm is None:
                 raise error.internal('source macro not found: %s in %s (%s)' % \
                                          (s, name, _map))
-            url = self.config.expand(sm[2])
+            opts = []
-            src = download.parse_url(url, '_sourcedir', self.config, self.opts)
+            url = []
+            for sp in sm[2].split():
+                if len(url) == 0 and sp[0] == '-':
+                    opts += [sp]
+                else:
+                    url += [sp]
+            if len(url) == 0:
+                raise error.general('source URL not found: %s' % (' '.join(args)))
+            #
+            # Look for --rsb-file as an option we use as a local file name.
+            # This can be used if a URL has no reasonable file name the
+            # download URL parser can figure out.
+            #
+            file_override = None
+            if len(opts) > 0:
+                for o in opts:
+                    if o.startswith('--rsb-file'):
+                       os_ = o.split('=')
+                       if len(os_) != 2:
+                           raise error.general('invalid --rsb-file option: %s' % (' '.join(args)))
+                       if os_[0] != '--rsb-file':
+                           raise error.general('invalid --rsb-file option: %s' % (' '.join(args)))
+                       file_override = os_[1]
+                opts = [o for o in opts if not o.startswith('--rsb-')]
+            url = self.config.expand(' '.join(url))
+            src = download.parse_url(url, '_sourcedir', self.config, self.opts, file_override)
             download.get_file(src['url'], src['local'], self.opts, self.config)
             if 'symlink' in src:
                 sname = name.replace('-', '_')
@@ -303,6 +328,22 @@ class build:
                     url += [pp]
             if len(url) == 0:
                 raise error.general('patch URL not found: %s' % (' '.join(args)))
+            #
+            # Look for --rsb-file as an option we use as a local file name.
+            # This can be used if a URL has no reasonable file name the
+            # download URL parser can figure out.
+            #
+            file_override = None
+            if len(opts) > 0:
+                for o in opts:
+                    if o.startswith('--rsb-file'):
+                       os_ = o.split('=')
+                       if len(os_) != 2:
+                           raise error.general('invalid --rsb-file option: %s' % (' '.join(args)))
+                       if os_[0] != '--rsb-file':
+                           raise error.general('invalid --rsb-file option: %s' % (' '.join(args)))
+                       file_override = os_[1]
+                opts = [o for o in opts if not o.startswith('--rsb-')]
             if len(opts) == 0:
                 opts = default_opts
             else:
@@ -312,12 +353,10 @@ class build:
             #
             # Parse the URL first in the source builder's patch directory.
             #
-            patch = download.parse_url(url, '_patchdir', self.config, self.opts)
+            patch = download.parse_url(url, '_patchdir', self.config, self.opts, file_override)
             #
-            # If not in the source builder package check the source directory.
+            # Download the patch
             #
-            if not path.exists(patch['local']):
-                patch = download.parse_url(url, '_patchdir', self.config, self.opts)
             download.get_file(patch['url'], patch['local'], self.opts, self.config)
             if 'compressed' in patch:
                 patch['script'] = patch['compressed'] + ' ' +  patch['local']

source-builder/sb/download.py

@@ -26,6 +26,7 @@ from __future__ import print_function
 
 import hashlib
 import os
+import re
 import stat
 import sys
 try:
@@ -130,57 +131,62 @@ def _hash_check(file_, absfile, macros, remove = True):
 
 def _local_path(source, pathkey, config):
     for p in config.define(pathkey).split(':'):
-        local = path.join(path.abspath(p), source['file'])
+        local_prefix = path.abspath(p)
+        local = path.join(local_prefix, source['file'])
         if source['local'] is None:
-            source['local_prefix'] = path.abspath(p)
+            source['local_prefix'] = local_prefix
             source['local'] = local
         if path.exists(local):
-            source['local_prefix'] = path.abspath(p)
+            source['local_prefix'] = local_prefix
             source['local'] = local
             _hash_check(source['file'], local, config.macros)
             break
 
 def _http_parser(source, pathkey, config, opts):
     #
-    # Hack for gitweb.cgi patch downloads. We rewrite the various fields.
+    # If the file has not been overrided attempt to recover a possible file name.
     #
-    if 'gitweb.cgi' in source['url']:
+    if 'file-override' not in source['options']:
-        url = source['url']
-        if '?' not in url:
-            raise error.general('invalid gitweb.cgi request: %s' % (url))
-        req = url.split('?')[1]
-        if len(req) == 0:
-            raise error.general('invalid gitweb.cgi request: %s' % (url))
         #
-        # The gitweb.cgi request should have:
+        # Hack for gitweb.cgi patch downloads. We rewrite the various fields.
-        #    p=<what>
-        #    a=patch
-        #    h=<hash>
-        # so extract the p and h parts to make the local name.
         #
-        p = None
+        if 'gitweb.cgi' in source['url']:
-        a = None
+            url = source['url']
-        h = None
+            if '?' not in url:
-        for r in req.split(';'):
+                raise error.general('invalid gitweb.cgi request: %s' % (url))
-            if '=' not in r:
+            req = url.split('?')[1]
-                raise error.general('invalid gitweb.cgi path: %s' % (url))
+            if len(req) == 0:
-            rs = r.split('=')
+                raise error.general('invalid gitweb.cgi request: %s' % (url))
-            if rs[0] == 'p':
+            #
-                p = rs[1].replace('.', '-')
+            # The gitweb.cgi request should have:
-            elif rs[0] == 'a':
+            #    p=<what>
-                a = rs[1]
+            #    a=patch
-            elif rs[0] == 'h':
+            #    h=<hash>
-                h = rs[1]
+            # so extract the p and h parts to make the local name.
-        if p is None or h is None:
+            #
-            raise error.general('gitweb.cgi path missing p or h: %s' % (url))
+            p = None
-        source['file'] = '%s-%s.patch' % (p, h)
+            a = None
-    #
+            h = None
-    # Check the source file name for any extra request query data and remove if
+            for r in req.split(';'):
-    # found. Some hosts do not like file names containing them.
+                if '=' not in r:
-    #
+                    raise error.general('invalid gitweb.cgi path: %s' % (url))
-    if '?' in source['file']:
+                rs = r.split('=')
-        qmark = source['file'].find('?')
+                if rs[0] == 'p':
-        source['file'] = source['file'][:qmark]
+                    p = rs[1].replace('.', '-')
+                elif rs[0] == 'a':
+                    a = rs[1]
+                elif rs[0] == 'h':
+                    h = rs[1]
+            if p is None or h is None:
+                raise error.general('gitweb.cgi path missing p or h: %s' % (url))
+            source['file'] = '%s-%s.patch' % (p, h)
+        #
+        # Check the source file name for any extra request query data and remove if
+        # found. Some hosts do not like file names containing them.
+        #
+        if '?' in source['file']:
+            qmark = source['file'].find('?')
+            source['file'] = source['file'][:qmark]
     #
     # Check local path
     #
@@ -301,17 +307,27 @@ def set_release_path(release_path, macros):
         release_path = '%{rtems_release_url}/%{rsb_version}/sources'
     macros.define('release_path', release_path)
 
-def parse_url(url, pathkey, config, opts):
+def parse_url(url, pathkey, config, opts, file_override = None):
     #
     # Split the source up into the parts we need.
     #
     source = {}
     source['url'] = url
+    source['options'] = []
     colon = url.find(':')
     if url[colon + 1:colon + 3] != '//':
         raise error.general('malforned URL (no protocol prefix): %s' % (url))
     source['path'] = url[:colon + 3] + path.dirname(url[colon + 3:])
-    source['file'] = path.basename(url)
+    if file_override is None:
+        source['file'] = path.basename(url)
+    else:
+        bad_chars = [c for c in ['/', '\\', '?', '*'] if c in file_override]
+        if len(bad_chars) > 0:
+            raise error.general('bad characters in file name: %s' % (file_override))
+
+        log.output('download: file-override: %s' % (file_override))
+        source['file'] = file_override
+        source['options'] += ['file-override']
     source['name'], source['ext'] = path.splitext(source['file'])
     if source['name'].endswith('.tar'):
         source['name'] = source['name'][:-4]
@@ -337,7 +353,7 @@ def _http_downloader(url, local, config, opts):
     if url.startswith('https://api.github.com'):
         url = urllib_parse.urljoin(url, config.expand('tarball/%{version}'))
     dst = os.path.relpath(path.host(local))
-    log.output('download: %s -> %s' % (url, dst))
+    log.output('download: (full) %s -> %s' % (url, dst))
     log.notice('download: %s -> %s' % (_sensible_url(url, len(dst)), dst))
     failed = False
     if _do_download(opts):
@@ -363,6 +379,7 @@ def _http_downloader(url, local, config, opts):
                     _ssl_context = ssl._create_unverified_context()
                     _in = urllib_request.urlopen(_req, context = _ssl_context)
                 except:
+                    log.output('download: no ssl context')
                     _ssl_context = None
                 if _ssl_context is None:
                     _in = urllib_request.urlopen(_req)