Add checksum support for downloaded files.

File download by http, ftp, pw support checksum. The %hash
directive provides a means of setting a hash used to
checksum the file.

Files on disk or just downloaded are checked.

bare/config/devel/expat-2.1.0-1.cfg

@@ -10,6 +10,8 @@
 
 %define expat_version 2.1.0
 
+%hash md5 expat-%{expat_version}.tar.gz dd7dab7a5fea97d2a6a43f511449b7cd
+
 #
 # The Expat build instructions. We use 2.xx Release 1.
 #

rtems/config/4.11/rtems-arm.bset

@@ -20,6 +20,7 @@
 # Add support for Cortex-M4 and Cortex-R based chips
 #
 %patch add gcc pw://patchwork.ozlabs.org/patch/372178/raw/v2-RTEMS-Add-multilibs-for-ARM.patch
+%hash  md5 v2-RTEMS-Add-multilibs-for-ARM.patch 2f3b5cf8db2a4c19f77752851ca10f6e
 
 #
 # Tools configuration.

rtems/config/4.11/rtems-lm32.bset

@@ -15,6 +15,7 @@
 # Fix the simulator UART interface.
 #
 %patch add gdb %{rtems_gdb_patches}/lm32/gdb-sim-lm32uart.diff
+%hash  md5 gdb-sim-lm32uart.diff 77d070878112783292461bd6e7db17fb
 
 #
 # Tools configuration.
@@ -22,6 +23,6 @@
 4.11/rtems-autotools
 devel/expat-2.1.0-1
 tools/rtems-binutils-2.24-1
-tools/rtems-gcc-4.9.0-newlib-cvs-1
+tools/rtems-gcc-4.9.1-newlib-cvs-1
 tools/rtems-gdb-7.7-1
 tools/rtems-kernel-4.11

rtems/config/4.11/rtems-microblaze.bset

@@ -15,11 +15,13 @@
 # GCC 4.7.2 Microblaze Patch
 #
 %patch add gcc %{rtems_gcc_patches}/microblaze/gcc-4.7.2-microblaze-ldscript-2.diff
+%hash  md5 gcc-4.7.2-microblaze-ldscript-2.diff a34f87dfa792193ade7f95e0b084a262
 
 #
 # GCC 4.8.2 Microblaze Patch for libgcc.
 #
 %patch add gcc %{rtems_gcc_patches}/microblaze/rtems-gcc-microblaze-libgcc-20131017.diff
+%hash  md5 rtems-gcc-microblaze-libgcc-20131017.diff 097186bb4c2c9a1970fe502aec30530f
 
 #
 # Tools configuration.

rtems/config/4.11/rtems-nios2.bset

@@ -17,6 +17,7 @@
 # Add support for Cortex-M4 and Cortex-R based chips
 #
 %patch add gcc pw://patchwork.ozlabs.org/patch/364504/raw/RTEMS-Add-Nios-2-support.patch
+%hash  md5 RTEMS-Add-Nios-2-support.patch fd9bbc8482e69623214e2a4289e38d8f
 
 #
 # Tools configuration.

rtems/config/4.11/rtems-sparc.bset

@@ -15,6 +15,7 @@
 # GDB patches
 #
 %patch add gdb %{rtems_gdb_patches}/sparc/gdb-7.7-sis-output-nouartrx.diff
+%hash  md5 gdb-7.7-sis-output-nouartrx.diff bc35b15de1325f3c4461ac3cda027bea
 
 #
 # Tools configuration.

rtems/config/net-mgmt/net-snmp-5.7.2.1-1.cfg

@@ -13,6 +13,8 @@
 #
 %define net_snmp_version 5.7.2.1
 
+%hash md5 net-snmp-%{net_snmp_version}.tar.gz 7db683faba037249837b226f64d566d4
+
 #
 # We need some special flags to build this version.
 #
@@ -22,6 +24,7 @@
 # Patch for RTEMS support.
 #
 %patch add net-snmp %{rtems_git_tools}/net-snmp/rtems-net-snmp-5.7.2.1-20140623.patch
+%hash  md5 rtems-net-snmp-5.7.2.1-20140623.patch 9a22fb4366a113f5df71ed2acd672b2b
 
 #
 # NetSNMP Build configuration

rtems/config/net/ntp-4.2.6p5-1.cfg

@@ -13,10 +13,13 @@
 #
 %define ntp_version 4.2.6p5
 
+%hash md5 ntp-%{ntp_version}.tar.gz 00df80a84ec9528fcfb09498075525bc
+
 #
 # Patch for RTEMS support.
 #
 %patch add ntp %{rtems_git_tools}/ntp/rtems-ntp-4.2.6p5.diff
+%hash  md5 rtems-ntp-4.2.6p5.diff 6fce685a73873f924a48b19d6b1700fc
 
 #
 # NTP Build configuration

rtems/config/tools/rtems-autoconf-2.69-1.cfg

@@ -8,6 +8,8 @@
 
 %define rtems_autoconf_version 2.69
 
+%hash md5 autoconf-%{rtems_autoconf_version}.tar.gz 82d05e03b93e45f5a39b828dc9c6c29b
+
 #
 # Check the version of autoconf in the path.
 #

rtems/config/tools/rtems-automake-1.12.6-1.cfg

@@ -8,6 +8,8 @@
 
 %define rtems_automake_version 1.12.6
 
+%hash md5 automake-%{rtems_automake_version}.tar.gz 199d39ece2e6070d64ac20d45ac86026
+
 #
 # Check the version of automake in the path.
 #

rtems/config/tools/rtems-binutils-2.24-1.cfg

@@ -7,6 +7,8 @@
 
 %define binutils_version 2.24
 
+%hash md5 binutils-%{binutils_version}.tar.bz2 e0f71a7b2ddab0f8612336ac81d9636b
+
 #
 # Enable deterministic archives by default. This will be the default
 # there all tools using this binutils will create deterministic

rtems/config/tools/rtems-gcc-4.8-newlib-cvs-1.cfg

@@ -25,6 +25,13 @@
 %define with_iconv 1
 %endif
 
+#
+# Hash checksums of the libraries used in GCC.
+#
+%hash md5 mpfr-%{mpfr_version}.tar.bz2 bfbecb2eacb6d48432ead5cfc3f7390a
+%hash md5 mpc-%{mpc_version}.tar.gz    e98267ebd5648a39f881d66797122fb6
+%hash md5 gmp-%{gmp_version}.tar.bz2   041487d25e9c230b0c42b106361055fe
+
 #
 # Newlib is from CVS.
 #
@@ -34,6 +41,7 @@
 # Joel's stack_t patch.
 #
 %patch add newlib -p0 %{rtems_newlib_patches}/newlib-sys-signal-20130532.diff
+%hash  md5 newlib-sys-signal-20130532.diff 34af05f28a4921e47c053528f5a8b9fb
 
 #
 # The gcc/newlib build instructions. We use 4.8 Release 1.

rtems/config/tools/rtems-gcc-4.8.2-newlib-2.1.0-1.cfg

@@ -19,10 +19,20 @@
 %define with_iconv 1
 %endif
 
+#
+# Hash checksums.
+#
+%hash md5 mpfr-%{mpfr_version}.tar.bz2    bfbecb2eacb6d48432ead5cfc3f7390a
+%hash md5 mpc-%{mpc_version}.tar.gz       e98267ebd5648a39f881d66797122fb6
+%hash md5 gmp-%{gmp_version}.tar.bz2      041487d25e9c230b0c42b106361055fe
+%hash md5 gcc-%{gcc_version}.tar.bz2      a3d7d63b9cb6b6ea049469a0c4a43c9d
+%hash md5 newlib-%{newlib_version}.tar.gz c6559d83ecce4728a52f0ce7ec80de97
+
 #
 # Joel's stack_t patch.
 #
 %patch add newlib -p0 %{rtems_newlib_patches}/newlib-sys-signal-20130532.diff
+%hash  md5 newlib-sys-signal-20130532.diff 34af05f28a4921e47c053528f5a8b9fb
 
 # The gcc/newlib build instructions. We use 4.8 Release 1.
 #

rtems/config/tools/rtems-gcc-4.8.2-newlib-cvs-1.cfg

@@ -5,6 +5,8 @@
 %define gcc_version    4.8.2
 %define newlib_version 12-Jul-2014
 
+%hash md5 gcc-%{gcc_version}.tar.bz2 a3d7d63b9cb6b6ea049469a0c4a43c9d
+
 #
 # The gcc/newlib build instructions.
 #

rtems/config/tools/rtems-gcc-4.8.3-newlib-cvs-1.cfg

@@ -5,6 +5,8 @@
 %define gcc_version    4.8.3
 %define newlib_version 26-Jul-2014
 
+%hash md5 gcc-%{gcc_version}.tar.bz2 7c60f24fab389f77af203d2516ee110f
+
 #
 # The gcc/newlib build instructions.
 #

rtems/config/tools/rtems-gcc-4.9-newlib-cvs-1.cfg

@@ -26,7 +26,14 @@
 %endif
 
 #
-# Newlib is from CVS.
+# Hash checksums of the libraries used in GCC.
+#
+%hash md5 mpfr-%{mpfr_version}.tar.bz2 bfbecb2eacb6d48432ead5cfc3f7390a
+%hash md5 mpc-%{mpc_version}.tar.gz    e98267ebd5648a39f881d66797122fb6
+%hash md5 gmp-%{gmp_version}.tar.bz2   041487d25e9c230b0c42b106361055fe
+
+#
+# Newlib is from CVS. No hash on CVS.
 #
 %source set newlib cvs://pserver:anoncvs@sourceware.org/cvs/src?module=newlib?src-prefix=src?date=%{newlib_version}?reset
 
@@ -34,6 +41,7 @@
 # Joel's stack_t patch.
 #
 %patch add newlib -p0 %{rtems_newlib_patches}/newlib-sys-signal-20130532.diff
+%hash  md5 newlib-sys-signal-20130532.diff 34af05f28a4921e47c053528f5a8b9fb
 
 #
 # The gcc/newlib build instructions. We use 4.9 Release 1.

rtems/config/tools/rtems-gcc-4.9.0-newlib-2.1.0-1.cfg

@@ -19,10 +19,20 @@
 %define with_iconv 1
 %endif
 
+#
+# Hash checksums.
+#
+%hash md5 mpfr-%{mpfr_version}.tar.bz2    bfbecb2eacb6d48432ead5cfc3f7390a
+%hash md5 mpc-%{mpc_version}.tar.gz       e98267ebd5648a39f881d66797122fb6
+%hash md5 gmp-%{gmp_version}.tar.bz2      041487d25e9c230b0c42b106361055fe
+%hash md5 gcc-%{gcc_version}.tar.bz2      9709b49ae0e904cbb0a6a1b62853b556
+%hash md5 newlib-%{newlib_version}.tar.gz c6559d83ecce4728a52f0ce7ec80de97
+
 #
 # Joel's stack_t patch.
 #
 %patch add newlib -p0 %{rtems_newlib_patches}/newlib-sys-signal-20130532.diff
+%hash  md5 newlib-sys-signal-20130532.diff 34af05f28a4921e47c053528f5a8b9fb
 
 #
 # The gcc/newlib build instructions. We use 4.9 Release 1.

rtems/config/tools/rtems-gcc-4.9.1-newlib-cvs-1.cfg

@@ -5,6 +5,8 @@
 %define gcc_version    4.9.1
 %define newlib_version 26-Jul-2014
 
+%hash md5 gcc-%{gcc_version}.tar.bz2 fddf71348546af523353bd43d34919c1
+
 #
 # The gcc/newlib build instructions.
 #

rtems/config/tools/rtems-gdb-7.7-1.cfg

@@ -7,14 +7,19 @@
 
 %define gdb_version 7.7
 
+%hash md5 gdb-%{gdb_version}.tar.bz2 271a18f41858a7e98b28ae4eb91287c9
+
 #
 # Clean up the sim-arange inline code so it builds.
 #
 %patch add gdb %{rtems_gdb_patches}/gdb-sim-arange-inline.diff
+%hash  md5 gdb-sim-arange-inline.diff 11bb2936ea29afeaa023077191fd4705
 %patch add gdb %{rtems_gdb_patches}/gdb-sim-cgen-inline.diff
+%hash  md5 gdb-sim-cgen-inline.diff e6f7d6d7295cdba99f51aab514ea9778
 
 %if %{_build_os} == freebsd
  %patch add gdb -p0 %{rtems_gdb_patches}/patch-gdb-python-python-config.py
+ %hash  md5 patch-gdb-python-python-config.py c0260fcca4c1a5509635049c0094eee3
 %endif
 
 #

rtems/config/tools/rtems-gdb-7.7.1-1.cfg

@@ -11,10 +11,13 @@
 # Clean up the sim-arange inline code so it builds.
 #
 %patch add gdb %{rtems_gdb_patches}/gdb-sim-arange-inline.diff
+%hash  md5 gdb-sim-arange-inline.diff 11bb2936ea29afeaa023077191fd4705
 %patch add gdb %{rtems_gdb_patches}/gdb-sim-cgen-inline.diff
+%hash  md5 gdb-sim-cgen-inline.diff e6f7d6d7295cdba99f51aab514ea9778
 
 %if %{_build_os} == freebsd
  %patch add gdb -p0 %{rtems_gdb_patches}/patch-gdb-python-python-config.py
+ %hash  md5 patch-gdb-python-python-config.py c0260fcca4c1a5509635049c0094eee3
 %endif
 
 #

rtems/config/tools/rtems-nios2-binutils-2.20-1.cfg

@@ -16,7 +16,9 @@
 # Source
 #
 %source set binutils http://ftp.rtems.org/ftp/pub/rtems/people/chrisj/nios/nios2-binutils-2.20.tar.bz2
+%hash   md5 nios2-binutils-2.20.tar.bz2 42ce11b421a1d35fc7b17ee051ccf92a
 %patch  add binutils %{rtems_binutils_patches}/nios2/nios2-binutils-2.20-rtems-20110703.diff
+%hash   md5 nios2-binutils-2.20-rtems-20110703.diff edaa78cdf7b87866c976232ccb1704ac
 
 #
 # The binutils build instructions. We use 2.xx Release 1.

rtems/config/tools/rtems-nios2-gcc-4.1-newlib-1.19.0-1.cfg

@@ -42,11 +42,13 @@ BuildRoot: %{_tmppath}/%{name}-root-%(%{__id_u} -n)
 # GCC core and G++
 #
 %source set gcc http://ftp.rtems.org/ftp/pub/rtems/people/chrisj/nios/nios2-gcc-4.1.tar.bz2
+%hash   md5 nios2-gcc-4.1.tar.bz2 717e5a25fda68d96d09353bcfa149cff
 
 #
 # Newlib
 #
 %source set newlib ftp://sourceware.org/pub/newlib/newlib-%{newlib_version}.tar.gz
+%hash   md5 newlib-%{newlib_version}.tar.gz 0966e19f03217db9e9076894b47e6601
 
 #
 # The GCC library directory

source-builder/sb/build.py

@@ -308,6 +308,8 @@ class build:
             for l in _prep:
                 args = l.split()
                 if len(args):
+                    def err(msg):
+                        raise error.general('%s: %s' % (package, msg))
                     if args[0] == '%setup':
                         if len(args) == 1:
                             raise error.general('invalid %%setup directive: %s' % (' '.join(args)))
@@ -315,8 +317,11 @@ class build:
                             self.source_setup(package, args[1:])
                         elif args[1] == 'patch':
                             self.patch_setup(package, args[1:])
-                    elif args[0].startswith('%patch'):
+                    elif args[0] in ['%patch', '%source']:
-                        self.patch(package, args)
+                        sources.process(args[0][1:], args[1:], self.macros, err)
+                    elif args[0] == '%hash':
+                        sources.hash(args[1:], self.macros, err)
+                        self.hash(package, args)
                     else:
                         self.script.append(' '.join(args))
 

source-builder/sb/config.py

@@ -229,8 +229,9 @@ class file:
 
     _ignore = [ re.compile('%setup'),
                 re.compile('%configure'),
-                re.compile('%source[0-9]*'),
+                re.compile('%source'),
-                re.compile('%patch[0-9]*'),
+                re.compile('%patch'),
+                re.compile('%hash'),
                 re.compile('%select'),
                 re.compile('%disable') ]
 
@@ -671,6 +672,9 @@ class file:
     def _sources(self, ls):
         return sources.process(ls[0][1:], ls[1:], self.macros, self._error)
 
+    def _hash(self, ls):
+        return sources.hash(ls[1:], self.macros, self._error)
+
     def _define(self, config, ls):
         if len(ls) <= 1:
             log.warning('invalid macro definition')
@@ -892,6 +896,11 @@ class file:
                         d = self._sources(ls)
                         if d is not None:
                             return ('data', d)
+                elif ls[0] == '%hash':
+                    if isvalid:
+                        d = self._hash(ls)
+                        if d is not None:
+                            return ('data', d)
                 elif ls[0] == '%patch':
                     if isvalid:
                         self._select(config, ls)

source-builder/sb/download.py

@@ -22,6 +22,7 @@
 # installed not to be package unless you run a packager around this.
 #
 
+import hashlib
 import os
 import stat
 import sys
@@ -50,6 +51,47 @@ def _humanize_bytes(bytes, precision = 1):
             break
     return '%.*f%s' % (precision, float(bytes) / factor, suffix)
 
+def _hash_check(file_, absfile, macros, remove = True):
+    failed = False
+    if file_.lower() in macros.map_keys('hashes'):
+        m1, m2, hash = macros.get(file_.lower(), globals = False, maps = 'hashes')
+        hash = hash.split()
+        if len(hash) != 2:
+            raise error.internal('invalid hash format: %s' % (file_))
+        if hash[0] not in hashlib.algorithms:
+            raise error.general('invalid hash algorithm for %s: %s' % (file_, hash[0]))
+        hasher = None
+        _in = None
+        try:
+            hasher = hashlib.new(hash[0])
+            _in = open(absfile, 'rb')
+            hasher.update(_in.read())
+        except IOError, err:
+            log.notice('hash: %s: read error: %s' % (file_, str(err)))
+            failed = True
+        except:
+            msg = 'hash: %s: error' % (file_)
+            log.stderr(msg)
+            log.notice(msg)
+            if _in is not None:
+                _in.close()
+            raise
+        if _in is not None:
+            _in.close()
+        log.output('checksums: %s: %s => %s' % (file_, hasher.hexdigest(), hash[1]))
+        if hasher.hexdigest() != hash[1]:
+            log.warning('checksum error: %s' % (file_))
+            failed = True
+        if failed and remove:
+            log.warning('removing: %s' % (file_))
+            if path.exists(absfile):
+                os.remove(path.host(absfile))
+        if hasher is not None:
+            del hasher
+    else:
+        log.warning('%s: no hash found' % (file_))
+    return not failed
+
 def _http_parser(source, config, opts):
     #
     # Is the file compressed ?
@@ -173,6 +215,7 @@ def parse_url(url, pathkey, config, opts):
         if path.exists(local):
             source['local_prefix'] = path.abspath(p)
             source['local'] = local
+            _hash_check(source['file'], local, config.macros)
             break
     source['script'] = ''
     for p in parsers:
@@ -257,6 +300,8 @@ def _http_downloader(url, local, config, opts):
         if not failed:
             if not path.isfile(local):
                 raise error.general('source is not a file: %s' % (path.host(local)))
+            if not _hash_check(path.basename(local), local, config.macros, False):
+                raise error.general('checksum failure file: %s' % (dst))
     return not failed
 
 def _git_downloader(url, local, config, opts):

source-builder/sb/log.py

@@ -82,7 +82,7 @@ def trace(text = os.linesep, log = None):
 
 def warning(text = os.linesep, log = None):
     for l in text.replace(chr(13), '').splitlines():
-        _output('warning: %s' % (l), log)
+        notice('warning: %s' % (l), log)
 
 def flush(log = None):
     if log:

source-builder/sb/setbuilder.py

@@ -236,6 +236,8 @@ class buildset:
                     self.bset_pkg = self.macros.expand(ls[1].strip())
                     self.macros['package'] = self.bset_pkg
                 elif ls[0][0] == '%':
+                    def err(msg):
+                        raise error.general('%s:%d: %s' % (self.bset, lc, msg))
                     if ls[0] == '%define':
                         if len(ls) > 2:
                             self.macros.define(ls[1].strip(),
@@ -249,10 +251,10 @@ class buildset:
                         self.macros.undefine(ls[1].strip())
                     elif ls[0] == '%include':
                         configs += self.parse(ls[1].strip())
-                    elif ls[0] == '%patch' or ls[0] == '%source':
+                    elif ls[0] in ['%patch', '%source']:
-                        def err(msg):
-                            raise error.general('%s:%d: %s' % (self.bset, lc, msg))
                         sources.process(ls[0][1:], ls[1:], self.macros, err)
+                    elif ls[0] == '%hash':
+                        sources.hash(ls[1:], self.macros, err)
                 else:
                     l = l.strip()
                     c = build.find_config(l, self.configs)

source-builder/sb/sources.py

@@ -72,3 +72,17 @@ def process(label, args, macros, error):
     elif args[0] == 'setup':
         return setup(label, args[1:], macros, error)
     error('invalid %%%s command: %s' % (label, args[0]))
+
+def hash(args, macros, error):
+    args = _args(args)
+    if len(args) != 3:
+        error('invalid number of hash args')
+    _map = 'hashes'
+    _file = macros.expand(args[1])
+    if _file in macros.map_keys(_map):
+        error('hash already set: %s' % (args[1]))
+    macros.create_map(_map)
+    macros.set_write_map(_map)
+    macros.define(_file, '%s %s' % (args[0], args[2]))
+    macros.unset_write_map()
+    return None