From a083b52921a86a3bc49e2cc1d7d102612694557c Mon Sep 17 00:00:00 2001 From: Chris Johns Date: Tue, 29 Jul 2014 16:35:43 +1000 Subject: [PATCH] Add checksum support for downloaded files. File download by http, ftp, pw support checksum. The %hash directive provides a means of setting a hash used to checksum the file. Files on disk or just downloaded are checked. --- bare/config/devel/expat-2.1.0-1.cfg | 2 + rtems/config/4.11/rtems-arm.bset | 1 + rtems/config/4.11/rtems-lm32.bset | 3 +- rtems/config/4.11/rtems-microblaze.bset | 2 + rtems/config/4.11/rtems-nios2.bset | 1 + rtems/config/4.11/rtems-sparc.bset | 1 + rtems/config/net-mgmt/net-snmp-5.7.2.1-1.cfg | 3 ++ rtems/config/net/ntp-4.2.6p5-1.cfg | 3 ++ rtems/config/tools/rtems-autoconf-2.69-1.cfg | 2 + .../config/tools/rtems-automake-1.12.6-1.cfg | 2 + rtems/config/tools/rtems-binutils-2.24-1.cfg | 2 + .../tools/rtems-gcc-4.8-newlib-cvs-1.cfg | 8 ++++ .../tools/rtems-gcc-4.8.2-newlib-2.1.0-1.cfg | 10 +++++ .../tools/rtems-gcc-4.8.2-newlib-cvs-1.cfg | 2 + .../tools/rtems-gcc-4.8.3-newlib-cvs-1.cfg | 2 + .../tools/rtems-gcc-4.9-newlib-cvs-1.cfg | 10 ++++- .../tools/rtems-gcc-4.9.0-newlib-2.1.0-1.cfg | 10 +++++ .../tools/rtems-gcc-4.9.1-newlib-cvs-1.cfg | 2 + rtems/config/tools/rtems-gdb-7.7-1.cfg | 5 +++ rtems/config/tools/rtems-gdb-7.7.1-1.cfg | 3 ++ .../tools/rtems-nios2-binutils-2.20-1.cfg | 2 + .../rtems-nios2-gcc-4.1-newlib-1.19.0-1.cfg | 2 + source-builder/sb/build.py | 9 +++- source-builder/sb/config.py | 13 +++++- source-builder/sb/download.py | 45 +++++++++++++++++++ source-builder/sb/log.py | 2 +- source-builder/sb/setbuilder.py | 8 ++-- source-builder/sb/sources.py | 14 ++++++ 28 files changed, 159 insertions(+), 10 deletions(-) diff --git a/bare/config/devel/expat-2.1.0-1.cfg b/bare/config/devel/expat-2.1.0-1.cfg index 7a87aab..fcb7cbe 100644 --- a/bare/config/devel/expat-2.1.0-1.cfg +++ b/bare/config/devel/expat-2.1.0-1.cfg @@ -10,6 +10,8 @@ %define expat_version 2.1.0 +%hash md5 expat-%{expat_version}.tar.gz dd7dab7a5fea97d2a6a43f511449b7cd + # # The Expat build instructions. We use 2.xx Release 1. # diff --git a/rtems/config/4.11/rtems-arm.bset b/rtems/config/4.11/rtems-arm.bset index c0fbe5d..61a565a 100644 --- a/rtems/config/4.11/rtems-arm.bset +++ b/rtems/config/4.11/rtems-arm.bset @@ -20,6 +20,7 @@ # Add support for Cortex-M4 and Cortex-R based chips # %patch add gcc pw://patchwork.ozlabs.org/patch/372178/raw/v2-RTEMS-Add-multilibs-for-ARM.patch +%hash md5 v2-RTEMS-Add-multilibs-for-ARM.patch 2f3b5cf8db2a4c19f77752851ca10f6e # # Tools configuration. diff --git a/rtems/config/4.11/rtems-lm32.bset b/rtems/config/4.11/rtems-lm32.bset index 9234ca9..6d34748 100644 --- a/rtems/config/4.11/rtems-lm32.bset +++ b/rtems/config/4.11/rtems-lm32.bset @@ -15,6 +15,7 @@ # Fix the simulator UART interface. # %patch add gdb %{rtems_gdb_patches}/lm32/gdb-sim-lm32uart.diff +%hash md5 gdb-sim-lm32uart.diff 77d070878112783292461bd6e7db17fb # # Tools configuration. @@ -22,6 +23,6 @@ 4.11/rtems-autotools devel/expat-2.1.0-1 tools/rtems-binutils-2.24-1 -tools/rtems-gcc-4.9.0-newlib-cvs-1 +tools/rtems-gcc-4.9.1-newlib-cvs-1 tools/rtems-gdb-7.7-1 tools/rtems-kernel-4.11 diff --git a/rtems/config/4.11/rtems-microblaze.bset b/rtems/config/4.11/rtems-microblaze.bset index 226e312..32c67b0 100644 --- a/rtems/config/4.11/rtems-microblaze.bset +++ b/rtems/config/4.11/rtems-microblaze.bset @@ -15,11 +15,13 @@ # GCC 4.7.2 Microblaze Patch # %patch add gcc %{rtems_gcc_patches}/microblaze/gcc-4.7.2-microblaze-ldscript-2.diff +%hash md5 gcc-4.7.2-microblaze-ldscript-2.diff a34f87dfa792193ade7f95e0b084a262 # # GCC 4.8.2 Microblaze Patch for libgcc. # %patch add gcc %{rtems_gcc_patches}/microblaze/rtems-gcc-microblaze-libgcc-20131017.diff +%hash md5 rtems-gcc-microblaze-libgcc-20131017.diff 097186bb4c2c9a1970fe502aec30530f # # Tools configuration. diff --git a/rtems/config/4.11/rtems-nios2.bset b/rtems/config/4.11/rtems-nios2.bset index f606099..bd39fe8 100644 --- a/rtems/config/4.11/rtems-nios2.bset +++ b/rtems/config/4.11/rtems-nios2.bset @@ -17,6 +17,7 @@ # Add support for Cortex-M4 and Cortex-R based chips # %patch add gcc pw://patchwork.ozlabs.org/patch/364504/raw/RTEMS-Add-Nios-2-support.patch +%hash md5 RTEMS-Add-Nios-2-support.patch fd9bbc8482e69623214e2a4289e38d8f # # Tools configuration. diff --git a/rtems/config/4.11/rtems-sparc.bset b/rtems/config/4.11/rtems-sparc.bset index 207a5fe..fd10efd 100644 --- a/rtems/config/4.11/rtems-sparc.bset +++ b/rtems/config/4.11/rtems-sparc.bset @@ -15,6 +15,7 @@ # GDB patches # %patch add gdb %{rtems_gdb_patches}/sparc/gdb-7.7-sis-output-nouartrx.diff +%hash md5 gdb-7.7-sis-output-nouartrx.diff bc35b15de1325f3c4461ac3cda027bea # # Tools configuration. diff --git a/rtems/config/net-mgmt/net-snmp-5.7.2.1-1.cfg b/rtems/config/net-mgmt/net-snmp-5.7.2.1-1.cfg index c86d16d..7527871 100644 --- a/rtems/config/net-mgmt/net-snmp-5.7.2.1-1.cfg +++ b/rtems/config/net-mgmt/net-snmp-5.7.2.1-1.cfg @@ -13,6 +13,8 @@ # %define net_snmp_version 5.7.2.1 +%hash md5 net-snmp-%{net_snmp_version}.tar.gz 7db683faba037249837b226f64d566d4 + # # We need some special flags to build this version. # @@ -22,6 +24,7 @@ # Patch for RTEMS support. # %patch add net-snmp %{rtems_git_tools}/net-snmp/rtems-net-snmp-5.7.2.1-20140623.patch +%hash md5 rtems-net-snmp-5.7.2.1-20140623.patch 9a22fb4366a113f5df71ed2acd672b2b # # NetSNMP Build configuration diff --git a/rtems/config/net/ntp-4.2.6p5-1.cfg b/rtems/config/net/ntp-4.2.6p5-1.cfg index 8cf02be..43afbf3 100644 --- a/rtems/config/net/ntp-4.2.6p5-1.cfg +++ b/rtems/config/net/ntp-4.2.6p5-1.cfg @@ -13,10 +13,13 @@ # %define ntp_version 4.2.6p5 +%hash md5 ntp-%{ntp_version}.tar.gz 00df80a84ec9528fcfb09498075525bc + # # Patch for RTEMS support. # %patch add ntp %{rtems_git_tools}/ntp/rtems-ntp-4.2.6p5.diff +%hash md5 rtems-ntp-4.2.6p5.diff 6fce685a73873f924a48b19d6b1700fc # # NTP Build configuration diff --git a/rtems/config/tools/rtems-autoconf-2.69-1.cfg b/rtems/config/tools/rtems-autoconf-2.69-1.cfg index 1db8746..acae2e9 100644 --- a/rtems/config/tools/rtems-autoconf-2.69-1.cfg +++ b/rtems/config/tools/rtems-autoconf-2.69-1.cfg @@ -8,6 +8,8 @@ %define rtems_autoconf_version 2.69 +%hash md5 autoconf-%{rtems_autoconf_version}.tar.gz 82d05e03b93e45f5a39b828dc9c6c29b + # # Check the version of autoconf in the path. # diff --git a/rtems/config/tools/rtems-automake-1.12.6-1.cfg b/rtems/config/tools/rtems-automake-1.12.6-1.cfg index 1585cf4..c9f3c28 100644 --- a/rtems/config/tools/rtems-automake-1.12.6-1.cfg +++ b/rtems/config/tools/rtems-automake-1.12.6-1.cfg @@ -8,6 +8,8 @@ %define rtems_automake_version 1.12.6 +%hash md5 automake-%{rtems_automake_version}.tar.gz 199d39ece2e6070d64ac20d45ac86026 + # # Check the version of automake in the path. # diff --git a/rtems/config/tools/rtems-binutils-2.24-1.cfg b/rtems/config/tools/rtems-binutils-2.24-1.cfg index 60cd982..ca6c02b 100644 --- a/rtems/config/tools/rtems-binutils-2.24-1.cfg +++ b/rtems/config/tools/rtems-binutils-2.24-1.cfg @@ -7,6 +7,8 @@ %define binutils_version 2.24 +%hash md5 binutils-%{binutils_version}.tar.bz2 e0f71a7b2ddab0f8612336ac81d9636b + # # Enable deterministic archives by default. This will be the default # there all tools using this binutils will create deterministic diff --git a/rtems/config/tools/rtems-gcc-4.8-newlib-cvs-1.cfg b/rtems/config/tools/rtems-gcc-4.8-newlib-cvs-1.cfg index 3ce89c2..5a75051 100644 --- a/rtems/config/tools/rtems-gcc-4.8-newlib-cvs-1.cfg +++ b/rtems/config/tools/rtems-gcc-4.8-newlib-cvs-1.cfg @@ -25,6 +25,13 @@ %define with_iconv 1 %endif +# +# Hash checksums of the libraries used in GCC. +# +%hash md5 mpfr-%{mpfr_version}.tar.bz2 bfbecb2eacb6d48432ead5cfc3f7390a +%hash md5 mpc-%{mpc_version}.tar.gz e98267ebd5648a39f881d66797122fb6 +%hash md5 gmp-%{gmp_version}.tar.bz2 041487d25e9c230b0c42b106361055fe + # # Newlib is from CVS. # @@ -34,6 +41,7 @@ # Joel's stack_t patch. # %patch add newlib -p0 %{rtems_newlib_patches}/newlib-sys-signal-20130532.diff +%hash md5 newlib-sys-signal-20130532.diff 34af05f28a4921e47c053528f5a8b9fb # # The gcc/newlib build instructions. We use 4.8 Release 1. diff --git a/rtems/config/tools/rtems-gcc-4.8.2-newlib-2.1.0-1.cfg b/rtems/config/tools/rtems-gcc-4.8.2-newlib-2.1.0-1.cfg index 33821a9..2c6e5e5 100644 --- a/rtems/config/tools/rtems-gcc-4.8.2-newlib-2.1.0-1.cfg +++ b/rtems/config/tools/rtems-gcc-4.8.2-newlib-2.1.0-1.cfg @@ -19,10 +19,20 @@ %define with_iconv 1 %endif +# +# Hash checksums. +# +%hash md5 mpfr-%{mpfr_version}.tar.bz2 bfbecb2eacb6d48432ead5cfc3f7390a +%hash md5 mpc-%{mpc_version}.tar.gz e98267ebd5648a39f881d66797122fb6 +%hash md5 gmp-%{gmp_version}.tar.bz2 041487d25e9c230b0c42b106361055fe +%hash md5 gcc-%{gcc_version}.tar.bz2 a3d7d63b9cb6b6ea049469a0c4a43c9d +%hash md5 newlib-%{newlib_version}.tar.gz c6559d83ecce4728a52f0ce7ec80de97 + # # Joel's stack_t patch. # %patch add newlib -p0 %{rtems_newlib_patches}/newlib-sys-signal-20130532.diff +%hash md5 newlib-sys-signal-20130532.diff 34af05f28a4921e47c053528f5a8b9fb # The gcc/newlib build instructions. We use 4.8 Release 1. # diff --git a/rtems/config/tools/rtems-gcc-4.8.2-newlib-cvs-1.cfg b/rtems/config/tools/rtems-gcc-4.8.2-newlib-cvs-1.cfg index 37ca021..c9e89b2 100644 --- a/rtems/config/tools/rtems-gcc-4.8.2-newlib-cvs-1.cfg +++ b/rtems/config/tools/rtems-gcc-4.8.2-newlib-cvs-1.cfg @@ -5,6 +5,8 @@ %define gcc_version 4.8.2 %define newlib_version 12-Jul-2014 +%hash md5 gcc-%{gcc_version}.tar.bz2 a3d7d63b9cb6b6ea049469a0c4a43c9d + # # The gcc/newlib build instructions. # diff --git a/rtems/config/tools/rtems-gcc-4.8.3-newlib-cvs-1.cfg b/rtems/config/tools/rtems-gcc-4.8.3-newlib-cvs-1.cfg index 2cbacd7..76a8310 100644 --- a/rtems/config/tools/rtems-gcc-4.8.3-newlib-cvs-1.cfg +++ b/rtems/config/tools/rtems-gcc-4.8.3-newlib-cvs-1.cfg @@ -5,6 +5,8 @@ %define gcc_version 4.8.3 %define newlib_version 26-Jul-2014 +%hash md5 gcc-%{gcc_version}.tar.bz2 7c60f24fab389f77af203d2516ee110f + # # The gcc/newlib build instructions. # diff --git a/rtems/config/tools/rtems-gcc-4.9-newlib-cvs-1.cfg b/rtems/config/tools/rtems-gcc-4.9-newlib-cvs-1.cfg index aedba21..3e89360 100644 --- a/rtems/config/tools/rtems-gcc-4.9-newlib-cvs-1.cfg +++ b/rtems/config/tools/rtems-gcc-4.9-newlib-cvs-1.cfg @@ -26,7 +26,14 @@ %endif # -# Newlib is from CVS. +# Hash checksums of the libraries used in GCC. +# +%hash md5 mpfr-%{mpfr_version}.tar.bz2 bfbecb2eacb6d48432ead5cfc3f7390a +%hash md5 mpc-%{mpc_version}.tar.gz e98267ebd5648a39f881d66797122fb6 +%hash md5 gmp-%{gmp_version}.tar.bz2 041487d25e9c230b0c42b106361055fe + +# +# Newlib is from CVS. No hash on CVS. # %source set newlib cvs://pserver:anoncvs@sourceware.org/cvs/src?module=newlib?src-prefix=src?date=%{newlib_version}?reset @@ -34,6 +41,7 @@ # Joel's stack_t patch. # %patch add newlib -p0 %{rtems_newlib_patches}/newlib-sys-signal-20130532.diff +%hash md5 newlib-sys-signal-20130532.diff 34af05f28a4921e47c053528f5a8b9fb # # The gcc/newlib build instructions. We use 4.9 Release 1. diff --git a/rtems/config/tools/rtems-gcc-4.9.0-newlib-2.1.0-1.cfg b/rtems/config/tools/rtems-gcc-4.9.0-newlib-2.1.0-1.cfg index 3d99702..3e6854e 100644 --- a/rtems/config/tools/rtems-gcc-4.9.0-newlib-2.1.0-1.cfg +++ b/rtems/config/tools/rtems-gcc-4.9.0-newlib-2.1.0-1.cfg @@ -19,10 +19,20 @@ %define with_iconv 1 %endif +# +# Hash checksums. +# +%hash md5 mpfr-%{mpfr_version}.tar.bz2 bfbecb2eacb6d48432ead5cfc3f7390a +%hash md5 mpc-%{mpc_version}.tar.gz e98267ebd5648a39f881d66797122fb6 +%hash md5 gmp-%{gmp_version}.tar.bz2 041487d25e9c230b0c42b106361055fe +%hash md5 gcc-%{gcc_version}.tar.bz2 9709b49ae0e904cbb0a6a1b62853b556 +%hash md5 newlib-%{newlib_version}.tar.gz c6559d83ecce4728a52f0ce7ec80de97 + # # Joel's stack_t patch. # %patch add newlib -p0 %{rtems_newlib_patches}/newlib-sys-signal-20130532.diff +%hash md5 newlib-sys-signal-20130532.diff 34af05f28a4921e47c053528f5a8b9fb # # The gcc/newlib build instructions. We use 4.9 Release 1. diff --git a/rtems/config/tools/rtems-gcc-4.9.1-newlib-cvs-1.cfg b/rtems/config/tools/rtems-gcc-4.9.1-newlib-cvs-1.cfg index 93e59e9..b8c9437 100644 --- a/rtems/config/tools/rtems-gcc-4.9.1-newlib-cvs-1.cfg +++ b/rtems/config/tools/rtems-gcc-4.9.1-newlib-cvs-1.cfg @@ -5,6 +5,8 @@ %define gcc_version 4.9.1 %define newlib_version 26-Jul-2014 +%hash md5 gcc-%{gcc_version}.tar.bz2 fddf71348546af523353bd43d34919c1 + # # The gcc/newlib build instructions. # diff --git a/rtems/config/tools/rtems-gdb-7.7-1.cfg b/rtems/config/tools/rtems-gdb-7.7-1.cfg index 3a89680..2f053de 100644 --- a/rtems/config/tools/rtems-gdb-7.7-1.cfg +++ b/rtems/config/tools/rtems-gdb-7.7-1.cfg @@ -7,14 +7,19 @@ %define gdb_version 7.7 +%hash md5 gdb-%{gdb_version}.tar.bz2 271a18f41858a7e98b28ae4eb91287c9 + # # Clean up the sim-arange inline code so it builds. # %patch add gdb %{rtems_gdb_patches}/gdb-sim-arange-inline.diff +%hash md5 gdb-sim-arange-inline.diff 11bb2936ea29afeaa023077191fd4705 %patch add gdb %{rtems_gdb_patches}/gdb-sim-cgen-inline.diff +%hash md5 gdb-sim-cgen-inline.diff e6f7d6d7295cdba99f51aab514ea9778 %if %{_build_os} == freebsd %patch add gdb -p0 %{rtems_gdb_patches}/patch-gdb-python-python-config.py + %hash md5 patch-gdb-python-python-config.py c0260fcca4c1a5509635049c0094eee3 %endif # diff --git a/rtems/config/tools/rtems-gdb-7.7.1-1.cfg b/rtems/config/tools/rtems-gdb-7.7.1-1.cfg index 55096e6..2e49f38 100644 --- a/rtems/config/tools/rtems-gdb-7.7.1-1.cfg +++ b/rtems/config/tools/rtems-gdb-7.7.1-1.cfg @@ -11,10 +11,13 @@ # Clean up the sim-arange inline code so it builds. # %patch add gdb %{rtems_gdb_patches}/gdb-sim-arange-inline.diff +%hash md5 gdb-sim-arange-inline.diff 11bb2936ea29afeaa023077191fd4705 %patch add gdb %{rtems_gdb_patches}/gdb-sim-cgen-inline.diff +%hash md5 gdb-sim-cgen-inline.diff e6f7d6d7295cdba99f51aab514ea9778 %if %{_build_os} == freebsd %patch add gdb -p0 %{rtems_gdb_patches}/patch-gdb-python-python-config.py + %hash md5 patch-gdb-python-python-config.py c0260fcca4c1a5509635049c0094eee3 %endif # diff --git a/rtems/config/tools/rtems-nios2-binutils-2.20-1.cfg b/rtems/config/tools/rtems-nios2-binutils-2.20-1.cfg index 4d4a4e6..f5c6329 100644 --- a/rtems/config/tools/rtems-nios2-binutils-2.20-1.cfg +++ b/rtems/config/tools/rtems-nios2-binutils-2.20-1.cfg @@ -16,7 +16,9 @@ # Source # %source set binutils http://ftp.rtems.org/ftp/pub/rtems/people/chrisj/nios/nios2-binutils-2.20.tar.bz2 +%hash md5 nios2-binutils-2.20.tar.bz2 42ce11b421a1d35fc7b17ee051ccf92a %patch add binutils %{rtems_binutils_patches}/nios2/nios2-binutils-2.20-rtems-20110703.diff +%hash md5 nios2-binutils-2.20-rtems-20110703.diff edaa78cdf7b87866c976232ccb1704ac # # The binutils build instructions. We use 2.xx Release 1. diff --git a/rtems/config/tools/rtems-nios2-gcc-4.1-newlib-1.19.0-1.cfg b/rtems/config/tools/rtems-nios2-gcc-4.1-newlib-1.19.0-1.cfg index 0ee5089..c6b48c1 100644 --- a/rtems/config/tools/rtems-nios2-gcc-4.1-newlib-1.19.0-1.cfg +++ b/rtems/config/tools/rtems-nios2-gcc-4.1-newlib-1.19.0-1.cfg @@ -42,11 +42,13 @@ BuildRoot: %{_tmppath}/%{name}-root-%(%{__id_u} -n) # GCC core and G++ # %source set gcc http://ftp.rtems.org/ftp/pub/rtems/people/chrisj/nios/nios2-gcc-4.1.tar.bz2 +%hash md5 nios2-gcc-4.1.tar.bz2 717e5a25fda68d96d09353bcfa149cff # # Newlib # %source set newlib ftp://sourceware.org/pub/newlib/newlib-%{newlib_version}.tar.gz +%hash md5 newlib-%{newlib_version}.tar.gz 0966e19f03217db9e9076894b47e6601 # # The GCC library directory diff --git a/source-builder/sb/build.py b/source-builder/sb/build.py index c868a46..d210259 100644 --- a/source-builder/sb/build.py +++ b/source-builder/sb/build.py @@ -308,6 +308,8 @@ class build: for l in _prep: args = l.split() if len(args): + def err(msg): + raise error.general('%s: %s' % (package, msg)) if args[0] == '%setup': if len(args) == 1: raise error.general('invalid %%setup directive: %s' % (' '.join(args))) @@ -315,8 +317,11 @@ class build: self.source_setup(package, args[1:]) elif args[1] == 'patch': self.patch_setup(package, args[1:]) - elif args[0].startswith('%patch'): - self.patch(package, args) + elif args[0] in ['%patch', '%source']: + sources.process(args[0][1:], args[1:], self.macros, err) + elif args[0] == '%hash': + sources.hash(args[1:], self.macros, err) + self.hash(package, args) else: self.script.append(' '.join(args)) diff --git a/source-builder/sb/config.py b/source-builder/sb/config.py index 5eb2d2a..baba751 100644 --- a/source-builder/sb/config.py +++ b/source-builder/sb/config.py @@ -229,8 +229,9 @@ class file: _ignore = [ re.compile('%setup'), re.compile('%configure'), - re.compile('%source[0-9]*'), - re.compile('%patch[0-9]*'), + re.compile('%source'), + re.compile('%patch'), + re.compile('%hash'), re.compile('%select'), re.compile('%disable') ] @@ -671,6 +672,9 @@ class file: def _sources(self, ls): return sources.process(ls[0][1:], ls[1:], self.macros, self._error) + def _hash(self, ls): + return sources.hash(ls[1:], self.macros, self._error) + def _define(self, config, ls): if len(ls) <= 1: log.warning('invalid macro definition') @@ -892,6 +896,11 @@ class file: d = self._sources(ls) if d is not None: return ('data', d) + elif ls[0] == '%hash': + if isvalid: + d = self._hash(ls) + if d is not None: + return ('data', d) elif ls[0] == '%patch': if isvalid: self._select(config, ls) diff --git a/source-builder/sb/download.py b/source-builder/sb/download.py index d774879..51747b1 100644 --- a/source-builder/sb/download.py +++ b/source-builder/sb/download.py @@ -22,6 +22,7 @@ # installed not to be package unless you run a packager around this. # +import hashlib import os import stat import sys @@ -50,6 +51,47 @@ def _humanize_bytes(bytes, precision = 1): break return '%.*f%s' % (precision, float(bytes) / factor, suffix) +def _hash_check(file_, absfile, macros, remove = True): + failed = False + if file_.lower() in macros.map_keys('hashes'): + m1, m2, hash = macros.get(file_.lower(), globals = False, maps = 'hashes') + hash = hash.split() + if len(hash) != 2: + raise error.internal('invalid hash format: %s' % (file_)) + if hash[0] not in hashlib.algorithms: + raise error.general('invalid hash algorithm for %s: %s' % (file_, hash[0])) + hasher = None + _in = None + try: + hasher = hashlib.new(hash[0]) + _in = open(absfile, 'rb') + hasher.update(_in.read()) + except IOError, err: + log.notice('hash: %s: read error: %s' % (file_, str(err))) + failed = True + except: + msg = 'hash: %s: error' % (file_) + log.stderr(msg) + log.notice(msg) + if _in is not None: + _in.close() + raise + if _in is not None: + _in.close() + log.output('checksums: %s: %s => %s' % (file_, hasher.hexdigest(), hash[1])) + if hasher.hexdigest() != hash[1]: + log.warning('checksum error: %s' % (file_)) + failed = True + if failed and remove: + log.warning('removing: %s' % (file_)) + if path.exists(absfile): + os.remove(path.host(absfile)) + if hasher is not None: + del hasher + else: + log.warning('%s: no hash found' % (file_)) + return not failed + def _http_parser(source, config, opts): # # Is the file compressed ? @@ -173,6 +215,7 @@ def parse_url(url, pathkey, config, opts): if path.exists(local): source['local_prefix'] = path.abspath(p) source['local'] = local + _hash_check(source['file'], local, config.macros) break source['script'] = '' for p in parsers: @@ -257,6 +300,8 @@ def _http_downloader(url, local, config, opts): if not failed: if not path.isfile(local): raise error.general('source is not a file: %s' % (path.host(local))) + if not _hash_check(path.basename(local), local, config.macros, False): + raise error.general('checksum failure file: %s' % (dst)) return not failed def _git_downloader(url, local, config, opts): diff --git a/source-builder/sb/log.py b/source-builder/sb/log.py index 2abcbe9..8e46023 100755 --- a/source-builder/sb/log.py +++ b/source-builder/sb/log.py @@ -82,7 +82,7 @@ def trace(text = os.linesep, log = None): def warning(text = os.linesep, log = None): for l in text.replace(chr(13), '').splitlines(): - _output('warning: %s' % (l), log) + notice('warning: %s' % (l), log) def flush(log = None): if log: diff --git a/source-builder/sb/setbuilder.py b/source-builder/sb/setbuilder.py index 7b285f2..6e7e7cc 100644 --- a/source-builder/sb/setbuilder.py +++ b/source-builder/sb/setbuilder.py @@ -236,6 +236,8 @@ class buildset: self.bset_pkg = self.macros.expand(ls[1].strip()) self.macros['package'] = self.bset_pkg elif ls[0][0] == '%': + def err(msg): + raise error.general('%s:%d: %s' % (self.bset, lc, msg)) if ls[0] == '%define': if len(ls) > 2: self.macros.define(ls[1].strip(), @@ -249,10 +251,10 @@ class buildset: self.macros.undefine(ls[1].strip()) elif ls[0] == '%include': configs += self.parse(ls[1].strip()) - elif ls[0] == '%patch' or ls[0] == '%source': - def err(msg): - raise error.general('%s:%d: %s' % (self.bset, lc, msg)) + elif ls[0] in ['%patch', '%source']: sources.process(ls[0][1:], ls[1:], self.macros, err) + elif ls[0] == '%hash': + sources.hash(ls[1:], self.macros, err) else: l = l.strip() c = build.find_config(l, self.configs) diff --git a/source-builder/sb/sources.py b/source-builder/sb/sources.py index 7b4edfd..d8f6f8d 100644 --- a/source-builder/sb/sources.py +++ b/source-builder/sb/sources.py @@ -72,3 +72,17 @@ def process(label, args, macros, error): elif args[0] == 'setup': return setup(label, args[1:], macros, error) error('invalid %%%s command: %s' % (label, args[0])) + +def hash(args, macros, error): + args = _args(args) + if len(args) != 3: + error('invalid number of hash args') + _map = 'hashes' + _file = macros.expand(args[1]) + if _file in macros.map_keys(_map): + error('hash already set: %s' % (args[1])) + macros.create_map(_map) + macros.set_write_map(_map) + macros.define(_file, '%s %s' % (args[0], args[2])) + macros.unset_write_map() + return None