diff --git a/tests/scripts/all-core.sh b/tests/scripts/all-core.sh new file mode 100644 index 0000000000..ccd7e59ea1 --- /dev/null +++ b/tests/scripts/all-core.sh @@ -0,0 +1,1023 @@ +# all-core.sh +# +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +################################################################ +#### Documentation +################################################################ + +# Purpose +# ------- +# +# To run all tests possible or available on the platform. +# +# Files structure +# --------------- +# +# The executable entry point for users and the CI is tests/scripts/all.sh. +# +# The actual content is in the following files: +# - all-core.sh contains the core logic for running test components, +# processing command line options, reporting results, etc. +# - all-helpers.sh contains helper functions used by more than 1 component. +# - components-*.sh contain the definitions of the various components. +# +# The first two parts are shared between repos and branches; +# the component files are repo&branch-specific. +# +# The files all-*.sh and components-*.sh should only define functions and not +# run code when sourced; the only exception being that all-core.sh runs +# 'shopt' because that is necessary for the rest of the file to parse. +# +# Notes for users +# --------------- +# +# Warning: the test is destructive. It includes various build modes and +# configurations, and can and will arbitrarily change the current CMake +# configuration. The following files must be committed into git: +# * include/mbedtls/mbedtls_config.h +# * Makefile, library/Makefile, programs/Makefile, tests/Makefile, +# programs/fuzz/Makefile +# After running this script, the CMake cache will be lost and CMake +# will no longer be initialised. +# +# The script assumes the presence of a number of tools: +# * Basic Unix tools (Windows users note: a Unix-style find must be before +# the Windows find in the PATH) +# * Perl +# * GNU Make +# * CMake +# * GCC and Clang (recent enough for using ASan with gcc and MemSan with clang, or valgrind) +# * G++ +# * arm-gcc and mingw-gcc +# * ArmCC 5 and ArmCC 6, unless invoked with --no-armcc +# * OpenSSL and GnuTLS command line tools, in suitable versions for the +# interoperability tests. The following are the official versions at the +# time of writing: +# * GNUTLS_{CLI,SERV} = 3.4.10 +# * GNUTLS_NEXT_{CLI,SERV} = 3.7.2 +# * OPENSSL = 1.0.2g (without Debian/Ubuntu patches) +# * OPENSSL_NEXT = 3.1.2 +# See the invocation of check_tools below for details. +# +# This script must be invoked from the toplevel directory of a git +# working copy of Mbed TLS. +# +# The behavior on an error depends on whether --keep-going (alias -k) +# is in effect. +# * Without --keep-going: the script stops on the first error without +# cleaning up. This lets you work in the configuration of the failing +# component. +# * With --keep-going: the script runs all requested components and +# reports failures at the end. In particular the script always cleans +# up on exit. +# +# Note that the output is not saved. You may want to run +# script -c tests/scripts/all.sh +# or +# tests/scripts/all.sh >all.log 2>&1 +# +# Notes for maintainers +# --------------------- +# +# The bulk of the code is organized into functions that follow one of the +# following naming conventions: +# * in all-core.sh: +# * pre_XXX: things to do before running the tests, in order. +# * post_XXX: things to do after running the tests. +# * in components-*.sh: +# * component_XXX: independent components. They can be run in any order. +# * component_check_XXX: quick tests that aren't worth parallelizing. +# * component_build_XXX: build things but don't run them. +# * component_test_XXX: build and test. +# * component_release_XXX: tests that the CI should skip during PR testing. +# * support_XXX: if support_XXX exists and returns false then +# component_XXX is not run by default. +# * in various files: +# * other: miscellaneous support functions. +# +# Each component must start by invoking `msg` with a short informative message. +# +# Warning: due to the way bash detects errors, the failure of a command +# inside 'if' or '!' is not detected. Use the 'not' function instead of '!'. +# +# Each component is executed in a separate shell process. The component +# fails if any command in it returns a non-zero status. +# +# The framework performs some cleanup tasks after each component. This +# means that components can assume that the working directory is in a +# cleaned-up state, and don't need to perform the cleanup themselves. +# * Run `make clean`. +# * Restore `include/mbedtls/mbedtls_config.h` from a backup made before running +# the component. +# * Check out `Makefile`, `library/Makefile`, `programs/Makefile`, +# `tests/Makefile` and `programs/fuzz/Makefile` from git. +# This cleans up after an in-tree use of CMake. + + +################################################################ +#### Initialization and command line parsing +################################################################ + +# Enable ksh/bash extended file matching patterns. +# Must come before function definitions or some of them wouldn't parse. +shopt -s extglob + +pre_set_shell_options () { + # Abort on errors (even on the left-hand side of a pipe). + # Treat uninitialised variables as errors. + set -e -o pipefail -u +} + +# For project detection +in_mbedtls_repo () { + test "$PROJECT_NAME" = "Mbed TLS" +} + +in_tf_psa_crypto_repo () { + test "$PROJECT_NAME" = "TF-PSA-Crypto" +} + +pre_check_environment () { + # For project detection + PROJECT_NAME_FILE='./scripts/project_name.txt' + if read -r PROJECT_NAME < "$PROJECT_NAME_FILE"; then :; else + echo "$PROJECT_NAME_FILE does not exist... Exiting..." >&2 + exit 1 + fi + + if in_mbedtls_repo || in_tf_psa_crypto_repo; then :; else + echo "Must be run from Mbed TLS / TF-PSA-Crypto root" >&2 + exit 1 + fi +} + +# Must be called before pre_initialize_variables which sets ALL_COMPONENTS. +pre_load_components () { + # Include the components from components.sh + test_script_dir="${0%/*}" + for file in "$test_script_dir"/components-*.sh; do + source $file + done +} + +pre_initialize_variables () { + if in_mbedtls_repo; then + CONFIG_H='include/mbedtls/mbedtls_config.h' + if [ -d tf-psa-crypto ]; then + CRYPTO_CONFIG_H='tf-psa-crypto/include/psa/crypto_config.h' + PSA_CORE_PATH='tf-psa-crypto/core' + BUILTIN_SRC_PATH='tf-psa-crypto/drivers/builtin/src' + else + CRYPTO_CONFIG_H='include/psa/crypto_config.h' + # helper_armc6_build_test() relies on these being defined, + # but empty if the paths don't exist (as in 3.6). + PSA_CORE_PATH='' + BUILTIN_SRC_PATH='' + fi + else + CONFIG_H='drivers/builtin/include/mbedtls/mbedtls_config.h' + CRYPTO_CONFIG_H='include/psa/crypto_config.h' + PSA_CORE_PATH='core' + BUILTIN_SRC_PATH='drivers/builtin/src' + fi + CONFIG_TEST_DRIVER_H='tests/include/test/drivers/config_test_driver.h' + + # Files that are clobbered by some jobs will be backed up. Use a different + # suffix from auxiliary scripts so that all.sh and auxiliary scripts can + # independently decide when to remove the backup file. + backup_suffix='.all.bak' + # Files clobbered by config.py + files_to_back_up="$CONFIG_H $CRYPTO_CONFIG_H $CONFIG_TEST_DRIVER_H" + if in_mbedtls_repo; then + # Files clobbered by in-tree cmake + files_to_back_up="$files_to_back_up Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile" + fi + + append_outcome=0 + MEMORY=0 + FORCE=0 + QUIET=0 + KEEP_GOING=0 + + # Seed value used with the --release-test option. + # + # See also RELEASE_SEED in basic-build-test.sh. Debugging is easier if + # both values are kept in sync. If you change the value here because it + # breaks some tests, you'll definitely want to change it in + # basic-build-test.sh as well. + RELEASE_SEED=1 + + # Specify character collation for regular expressions and sorting with C locale + export LC_COLLATE=C + + : ${MBEDTLS_TEST_OUTCOME_FILE=} + : ${MBEDTLS_TEST_PLATFORM="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"} + export MBEDTLS_TEST_OUTCOME_FILE + export MBEDTLS_TEST_PLATFORM + + # Default commands, can be overridden by the environment + : ${OPENSSL:="openssl"} + : ${OPENSSL_NEXT:="$OPENSSL"} + : ${GNUTLS_CLI:="gnutls-cli"} + : ${GNUTLS_SERV:="gnutls-serv"} + : ${OUT_OF_SOURCE_DIR:=./mbedtls_out_of_source_build} + : ${ARMC5_BIN_DIR:=/usr/bin} + : ${ARMC6_BIN_DIR:=/usr/bin} + : ${ARM_NONE_EABI_GCC_PREFIX:=arm-none-eabi-} + : ${ARM_LINUX_GNUEABI_GCC_PREFIX:=arm-linux-gnueabi-} + : ${CLANG_LATEST:="clang-latest"} + : ${CLANG_EARLIEST:="clang-earliest"} + : ${GCC_LATEST:="gcc-latest"} + : ${GCC_EARLIEST:="gcc-earliest"} + # if MAKEFLAGS is not set add the -j option to speed up invocations of make + if [ -z "${MAKEFLAGS+set}" ]; then + export MAKEFLAGS="-j$(all_sh_nproc)" + fi + # if CC is not set, use clang by default (if present) to improve build times + if [ -z "${CC+set}" ] && (type clang > /dev/null 2>&1); then + export CC="clang" + fi + + if [ -n "${OPENSSL_3+set}" ]; then + export OPENSSL_NEXT="$OPENSSL_3" + fi + + # Include more verbose output for failing tests run by CMake or make + export CTEST_OUTPUT_ON_FAILURE=1 + + # CFLAGS and LDFLAGS for Asan builds that don't use CMake + # default to -O2, use -Ox _after_ this if you want another level + ASAN_CFLAGS='-O2 -Werror -fsanitize=address,undefined -fno-sanitize-recover=all' + # Normally, tests should use this compiler for ASAN testing + ASAN_CC=clang + + # Platform tests have an allocation that returns null + export ASAN_OPTIONS="allocator_may_return_null=1" + export MSAN_OPTIONS="allocator_may_return_null=1" + + # Gather the list of available components. These are the functions + # defined in this script whose name starts with "component_". + ALL_COMPONENTS=$(compgen -A function component_ | sed 's/component_//') + + PSASIM_PATH='tests/psa-client-server/psasim/' + + # Delay determining SUPPORTED_COMPONENTS until the command line options have a chance to override + # the commands set by the environment +} + +setup_quiet_wrappers() +{ + # Pick up "quiet" wrappers for make and cmake, which don't output very much + # unless there is an error. This reduces logging overhead in the CI. + # + # Note that the cmake wrapper breaks unless we use an absolute path here. + if [[ -e ${PWD}/tests/scripts/quiet ]]; then + export PATH=${PWD}/tests/scripts/quiet:$PATH + fi +} + +# Test whether the component $1 is included in the command line patterns. +is_component_included() +{ + # Temporarily disable wildcard expansion so that $COMMAND_LINE_COMPONENTS + # only does word splitting. + set -f + for pattern in $COMMAND_LINE_COMPONENTS; do + set +f + case ${1#component_} in $pattern) return 0;; esac + done + set +f + return 1 +} + +usage() +{ + cat < + Prefix for a cross-compiler for arm-none-eabi + (default: "${ARM_NONE_EABI_GCC_PREFIX}") + --arm-linux-gnueabi-gcc-prefix= + Prefix for a cross-compiler for arm-linux-gnueabi + (default: "${ARM_LINUX_GNUEABI_GCC_PREFIX}") + --armcc Run ARM Compiler builds (on by default). + --restore First clean up the build tree, restoring backed up + files. Do not run any components unless they are + explicitly specified. + --error-test Error test mode: run a failing function in addition + to any specified component. May be repeated. + --except Exclude the COMPONENTs listed on the command line, + instead of running only those. + --no-append-outcome Write a new outcome file and analyze it (default). + --no-armcc Skip ARM Compiler builds. + --no-force Refuse to overwrite modified files (default). + --no-keep-going Stop at the first error (default). + --no-memory No additional memory tests (default). + --no-quiet Print full output from components. + --out-of-source-dir= Directory used for CMake out-of-source build tests. + --outcome-file= File where test outcomes are written (not done if + empty; default: \$MBEDTLS_TEST_OUTCOME_FILE). + --random-seed Use a random seed value for randomized tests (default). + -r|--release-test Run this script in release mode. This fixes the seed value to ${RELEASE_SEED}. + -s|--seed Integer seed value to use for this test run. + +Tool path options: + --armc5-bin-dir= ARM Compiler 5 bin directory. + --armc6-bin-dir= ARM Compiler 6 bin directory. + --clang-earliest= Earliest version of clang available + --clang-latest= Latest version of clang available + --gcc-earliest= Earliest version of GCC available + --gcc-latest= Latest version of GCC available + --gnutls-cli= GnuTLS client executable to use for most tests. + --gnutls-serv= GnuTLS server executable to use for most tests. + --openssl= OpenSSL executable to use for most tests. + --openssl-next= OpenSSL executable to use for recent things like ARIA +EOF +} + +# Cleanup before/after running a component. +# Remove built files as well as the cmake cache/config. +# Does not remove generated source files. +cleanup() +{ + if in_mbedtls_repo; then + command make clean + fi + + # Remove CMake artefacts + find . -name .git -prune -o \ + -iname CMakeFiles -exec rm -rf {} \+ -o \ + \( -iname cmake_install.cmake -o \ + -iname CTestTestfile.cmake -o \ + -iname CMakeCache.txt -o \ + -path './cmake/*.cmake' \) -exec rm -f {} \+ + # Remove Makefiles generated by in-tree CMake builds + # (Not all files will exist in all branches, but that's OK.) + rm -f 3rdparty/Makefile 3rdparty/*/Makefile + rm -f pkgconfig/Makefile framework/Makefile + rm -f include/Makefile programs/!(fuzz)/Makefile + rm -f tf-psa-crypto/Makefile tf-psa-crypto/include/Makefile + rm -f tf-psa-crypto/core/Makefile tf-psa-crypto/drivers/Makefile + rm -f tf-psa-crypto/tests/Makefile + rm -f tf-psa-crypto/drivers/everest/Makefile + rm -f tf-psa-crypto/drivers/p256-m/Makefile + rm -f tf-psa-crypto/drivers/builtin/Makefile + rm -f tf-psa-crypto/drivers/builtin/src/Makefile + + # Remove any artifacts from the component_test_cmake_as_subdirectory test. + rm -rf programs/test/cmake_subproject/build + rm -f programs/test/cmake_subproject/Makefile + rm -f programs/test/cmake_subproject/cmake_subproject + + # Remove any artifacts from the component_test_cmake_as_package test. + rm -rf programs/test/cmake_package/build + rm -f programs/test/cmake_package/Makefile + rm -f programs/test/cmake_package/cmake_package + + # Remove any artifacts from the component_test_cmake_as_installed_package test. + rm -rf programs/test/cmake_package_install/build + rm -f programs/test/cmake_package_install/Makefile + rm -f programs/test/cmake_package_install/cmake_package_install + + # Restore files that may have been clobbered by the job + restore_backed_up_files +} + +# Restore files that may have been clobbered +restore_backed_up_files () { + for x in $files_to_back_up; do + if [[ -e "$x$backup_suffix" ]]; then + cp -p "$x$backup_suffix" "$x" + fi + done +} + +# Final cleanup when this script exits (except when exiting on a failure +# in non-keep-going mode). +final_cleanup () { + cleanup + + for x in $files_to_back_up; do + rm -f "$x$backup_suffix" + done +} + +# Executed on exit. May be redefined depending on command line options. +final_report () { + : +} + +fatal_signal () { + final_cleanup + final_report $1 + trap - $1 + kill -$1 $$ +} + +pre_set_signal_handlers () { + trap 'fatal_signal HUP' HUP + trap 'fatal_signal INT' INT + trap 'fatal_signal TERM' TERM +} + +# Number of processors on this machine. Used as the default setting +# for parallel make. +all_sh_nproc () +{ + { + nproc || # Linux + sysctl -n hw.ncpuonline || # NetBSD, OpenBSD + sysctl -n hw.ncpu || # FreeBSD + echo 1 + } 2>/dev/null +} + +msg() +{ + if [ -n "${current_component:-}" ]; then + current_section="${current_component#component_}: $1" + else + current_section="$1" + fi + + if [ $QUIET -eq 1 ]; then + return + fi + + echo "" + echo "******************************************************************" + echo "* $current_section " + printf "* "; date + echo "******************************************************************" +} + +err_msg() +{ + echo "$1" >&2 +} + +check_tools() +{ + for tool in "$@"; do + if ! `type "$tool" >/dev/null 2>&1`; then + err_msg "$tool not found!" + exit 1 + fi + done +} + +pre_parse_command_line () { + COMMAND_LINE_COMPONENTS= + all_except=0 + error_test=0 + list_components=0 + restore_first=0 + no_armcc= + + # Note that legacy options are ignored instead of being omitted from this + # list of options, so invocations that worked with previous version of + # all.sh will still run and work properly. + while [ $# -gt 0 ]; do + case "$1" in + --append-outcome) append_outcome=1;; + --arm-none-eabi-gcc-prefix) shift; ARM_NONE_EABI_GCC_PREFIX="$1";; + --arm-linux-gnueabi-gcc-prefix) shift; ARM_LINUX_GNUEABI_GCC_PREFIX="$1";; + --armcc) no_armcc=;; + --armc5-bin-dir) shift; ARMC5_BIN_DIR="$1";; + --armc6-bin-dir) shift; ARMC6_BIN_DIR="$1";; + --clang-earliest) shift; CLANG_EARLIEST="$1";; + --clang-latest) shift; CLANG_LATEST="$1";; + --error-test) error_test=$((error_test + 1));; + --except) all_except=1;; + --force|-f) FORCE=1;; + --gcc-earliest) shift; GCC_EARLIEST="$1";; + --gcc-latest) shift; GCC_LATEST="$1";; + --gnutls-cli) shift; GNUTLS_CLI="$1";; + --gnutls-legacy-cli) shift;; # ignored for backward compatibility + --gnutls-legacy-serv) shift;; # ignored for backward compatibility + --gnutls-serv) shift; GNUTLS_SERV="$1";; + --help|-h) usage; exit;; + --keep-going|-k) KEEP_GOING=1;; + --list-all-components) printf '%s\n' $ALL_COMPONENTS; exit;; + --list-components) list_components=1;; + --memory|-m) MEMORY=1;; + --no-append-outcome) append_outcome=0;; + --no-armcc) no_armcc=1;; + --no-force) FORCE=0;; + --no-keep-going) KEEP_GOING=0;; + --no-memory) MEMORY=0;; + --no-quiet) QUIET=0;; + --openssl) shift; OPENSSL="$1";; + --openssl-next) shift; OPENSSL_NEXT="$1";; + --outcome-file) shift; MBEDTLS_TEST_OUTCOME_FILE="$1";; + --out-of-source-dir) shift; OUT_OF_SOURCE_DIR="$1";; + --quiet|-q) QUIET=1;; + --random-seed) unset SEED;; + --release-test|-r) SEED=$RELEASE_SEED;; + --restore) restore_first=1;; + --seed|-s) shift; SEED="$1";; + -*) + echo >&2 "Unknown option: $1" + echo >&2 "Run $0 --help for usage." + exit 120 + ;; + *) COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS $1";; + esac + shift + done + + # Exclude components that are not supported on this platform. + SUPPORTED_COMPONENTS= + for component in $ALL_COMPONENTS; do + case $(type "support_$component" 2>&1) in + *' function'*) + if ! support_$component; then continue; fi;; + esac + SUPPORTED_COMPONENTS="$SUPPORTED_COMPONENTS $component" + done + + if [ $list_components -eq 1 ]; then + printf '%s\n' $SUPPORTED_COMPONENTS + exit + fi + + # With no list of components, run everything. + if [ -z "$COMMAND_LINE_COMPONENTS" ] && [ $restore_first -eq 0 ]; then + all_except=1 + fi + + # --no-armcc is a legacy option. The modern way is --except '*_armcc*'. + # Ignore it if components are listed explicitly on the command line. + if [ -n "$no_armcc" ] && [ $all_except -eq 1 ]; then + COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS *_armcc*" + fi + + # Error out if an explicitly requested component doesn't exist. + if [ $all_except -eq 0 ]; then + unsupported=0 + # Temporarily disable wildcard expansion so that $COMMAND_LINE_COMPONENTS + # only does word splitting. + set -f + for component in $COMMAND_LINE_COMPONENTS; do + set +f + # If the requested name includes a wildcard character, don't + # check it. Accept wildcard patterns that don't match anything. + case $component in + *[*?\[]*) continue;; + esac + case " $SUPPORTED_COMPONENTS " in + *" $component "*) :;; + *) + echo >&2 "Component $component was explicitly requested, but is not known or not supported." + unsupported=$((unsupported + 1));; + esac + done + set +f + if [ $unsupported -ne 0 ]; then + exit 2 + fi + fi + + # Build the list of components to run. + RUN_COMPONENTS= + for component in $SUPPORTED_COMPONENTS; do + if is_component_included "$component"; [ $? -eq $all_except ]; then + RUN_COMPONENTS="$RUN_COMPONENTS $component" + fi + done + + unset all_except + unset no_armcc +} + +pre_check_git () { + if [ $FORCE -eq 1 ]; then + rm -rf "$OUT_OF_SOURCE_DIR" + git checkout-index -f -q $CONFIG_H + cleanup + else + + if [ -d "$OUT_OF_SOURCE_DIR" ]; then + echo "Warning - there is an existing directory at '$OUT_OF_SOURCE_DIR'" >&2 + echo "You can either delete this directory manually, or force the test by rerunning" + echo "the script as: $0 --force --out-of-source-dir $OUT_OF_SOURCE_DIR" + exit 1 + fi + + if ! git diff --quiet "$CONFIG_H"; then + err_msg "Warning - the configuration file '$CONFIG_H' has been edited. " + echo "You can either delete or preserve your work, or force the test by rerunning the" + echo "script as: $0 --force" + exit 1 + fi + fi +} + +pre_restore_files () { + # If the makefiles have been generated by a framework such as cmake, + # restore them from git. If the makefiles look like modifications from + # the ones checked into git, take care not to modify them. Whatever + # this function leaves behind is what the script will restore before + # each component. + case "$(head -n1 Makefile)" in + *[Gg]enerated*) + git update-index --no-skip-worktree Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile + git checkout -- Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile + ;; + esac +} + +pre_back_up () { + for x in $files_to_back_up; do + cp -p "$x" "$x$backup_suffix" + done +} + +pre_setup_keep_going () { + failure_count=0 # Number of failed components + last_failure_status=0 # Last failure status in this component + + # See err_trap + previous_failure_status=0 + previous_failed_command= + previous_failure_funcall_depth=0 + unset report_failed_command + + start_red= + end_color= + if [ -t 1 ]; then + case "${TERM:-}" in + *color*|cygwin|linux|rxvt*|screen|[Eex]term*) + start_red=$(printf '\033[31m') + end_color=$(printf '\033[0m') + ;; + esac + fi + + # Keep a summary of failures in a file. We'll print it out at the end. + failure_summary_file=$PWD/all-sh-failures-$$.log + : >"$failure_summary_file" + + # Whether it makes sense to keep a component going after the specified + # command fails (test command) or not (configure or build). + # This function normally receives the failing simple command + # ($BASH_COMMAND) as an argument, but if $report_failed_command is set, + # this is passed instead. + # This doesn't have to be 100% accurate: all failures are recorded anyway. + # False positives result in running things that can't be expected to + # work. False negatives result in things not running after something else + # failed even though they might have given useful feedback. + can_keep_going_after_failure () { + case "$1" in + "msg "*) false;; + "cd "*) false;; + "diff "*) true;; + *make*[\ /]tests*) false;; # make tests, make CFLAGS=-I../tests, ... + *test*) true;; # make test, tests/stuff, env V=v tests/stuff, ... + *make*check*) true;; + "grep "*) true;; + "[ "*) true;; + "! "*) true;; + *) false;; + esac + } + + # This function runs if there is any error in a component. + # It must either exit with a nonzero status, or set + # last_failure_status to a nonzero value. + err_trap () { + # Save $? (status of the failing command). This must be the very + # first thing, before $? is overridden. + last_failure_status=$? + failed_command=${report_failed_command-$BASH_COMMAND} + + if [[ $last_failure_status -eq $previous_failure_status && + "$failed_command" == "$previous_failed_command" && + ${#FUNCNAME[@]} == $((previous_failure_funcall_depth - 1)) ]] + then + # The same command failed twice in a row, but this time one level + # less deep in the function call stack. This happens when the last + # command of a function returns a nonzero status, and the function + # returns that same status. Ignore the second failure. + previous_failure_funcall_depth=${#FUNCNAME[@]} + return + fi + previous_failure_status=$last_failure_status + previous_failed_command=$failed_command + previous_failure_funcall_depth=${#FUNCNAME[@]} + + text="$current_section: $failed_command -> $last_failure_status" + echo "${start_red}^^^^$text^^^^${end_color}" >&2 + echo "$text" >>"$failure_summary_file" + + # If the command is fatal (configure or build command), stop this + # component. Otherwise (test command) keep the component running + # (run more tests from the same build). + if ! can_keep_going_after_failure "$failed_command"; then + exit $last_failure_status + fi + } + + final_report () { + if [ $failure_count -gt 0 ]; then + echo + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "${start_red}FAILED: $failure_count components${end_color}" + cat "$failure_summary_file" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + elif [ -z "${1-}" ]; then + echo "SUCCESS :)" + fi + if [ -n "${1-}" ]; then + echo "Killed by SIG$1." + fi + rm -f "$failure_summary_file" + if [ $failure_count -gt 0 ]; then + exit 1 + fi + } +} + +# '! true' does not trigger the ERR trap. Arrange to trigger it, with +# a reasonably informative error message (not just "$@"). +not () { + if "$@"; then + report_failed_command="! $*" + false + unset report_failed_command + fi +} + +pre_prepare_outcome_file () { + case "$MBEDTLS_TEST_OUTCOME_FILE" in + [!/]*) MBEDTLS_TEST_OUTCOME_FILE="$PWD/$MBEDTLS_TEST_OUTCOME_FILE";; + esac + if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ] && [ "$append_outcome" -eq 0 ]; then + rm -f "$MBEDTLS_TEST_OUTCOME_FILE" + fi +} + +pre_print_configuration () { + if [ $QUIET -eq 1 ]; then + return + fi + + msg "info: $0 configuration" + echo "MEMORY: $MEMORY" + echo "FORCE: $FORCE" + echo "MBEDTLS_TEST_OUTCOME_FILE: ${MBEDTLS_TEST_OUTCOME_FILE:-(none)}" + echo "SEED: ${SEED-"UNSET"}" + echo + echo "OPENSSL: $OPENSSL" + echo "OPENSSL_NEXT: $OPENSSL_NEXT" + echo "GNUTLS_CLI: $GNUTLS_CLI" + echo "GNUTLS_SERV: $GNUTLS_SERV" + echo "ARMC5_BIN_DIR: $ARMC5_BIN_DIR" + echo "ARMC6_BIN_DIR: $ARMC6_BIN_DIR" +} + +# Make sure the tools we need are available. +pre_check_tools () { + # Build the list of variables to pass to output_env.sh. + set env + + case " $RUN_COMPONENTS " in + # Require OpenSSL and GnuTLS if running any tests (as opposed to + # only doing builds). Not all tests run OpenSSL and GnuTLS, but this + # is a good enough approximation in practice. + *" test_"* | *" release_test_"*) + # To avoid setting OpenSSL and GnuTLS for each call to compat.sh + # and ssl-opt.sh, we just export the variables they require. + export OPENSSL="$OPENSSL" + export GNUTLS_CLI="$GNUTLS_CLI" + export GNUTLS_SERV="$GNUTLS_SERV" + # Avoid passing --seed flag in every call to ssl-opt.sh + if [ -n "${SEED-}" ]; then + export SEED + fi + set "$@" OPENSSL="$OPENSSL" + set "$@" GNUTLS_CLI="$GNUTLS_CLI" GNUTLS_SERV="$GNUTLS_SERV" + check_tools "$OPENSSL" "$OPENSSL_NEXT" \ + "$GNUTLS_CLI" "$GNUTLS_SERV" + ;; + esac + + case " $RUN_COMPONENTS " in + *_doxygen[_\ ]*) check_tools "doxygen" "dot";; + esac + + case " $RUN_COMPONENTS " in + *_arm_none_eabi_gcc[_\ ]*) check_tools "${ARM_NONE_EABI_GCC_PREFIX}gcc";; + esac + + case " $RUN_COMPONENTS " in + *_mingw[_\ ]*) check_tools "i686-w64-mingw32-gcc";; + esac + + case " $RUN_COMPONENTS " in + *" test_zeroize "*) check_tools "gdb";; + esac + + case " $RUN_COMPONENTS " in + *_armcc*) + ARMC5_CC="$ARMC5_BIN_DIR/armcc" + ARMC5_AR="$ARMC5_BIN_DIR/armar" + ARMC5_FROMELF="$ARMC5_BIN_DIR/fromelf" + ARMC6_CC="$ARMC6_BIN_DIR/armclang" + ARMC6_AR="$ARMC6_BIN_DIR/armar" + ARMC6_FROMELF="$ARMC6_BIN_DIR/fromelf" + check_tools "$ARMC5_CC" "$ARMC5_AR" "$ARMC5_FROMELF" \ + "$ARMC6_CC" "$ARMC6_AR" "$ARMC6_FROMELF";; + esac + + # past this point, no call to check_tool, only printing output + if [ $QUIET -eq 1 ]; then + return + fi + + msg "info: output_env.sh" + case $RUN_COMPONENTS in + *_armcc*) + set "$@" ARMC5_CC="$ARMC5_CC" ARMC6_CC="$ARMC6_CC" RUN_ARMCC=1;; + *) set "$@" RUN_ARMCC=0;; + esac + "$@" scripts/output_env.sh +} + +pre_generate_files() { + # since make doesn't have proper dependencies, remove any possibly outdate + # file that might be around before generating fresh ones + make neat + if [ $QUIET -eq 1 ]; then + make generated_files >/dev/null + else + make generated_files + fi +} + +pre_load_helpers () { + # The path is going to change when this is moved to the framework + test_script_dir="${0%/*}" + source "$test_script_dir"/all-helpers.sh +} + +################################################################ +#### Termination +################################################################ + +post_report () { + msg "Done, cleaning up" + final_cleanup + + final_report +} + +################################################################ +#### Run all the things +################################################################ + +# Function invoked by --error-test to test error reporting. +pseudo_component_error_test () { + msg "Testing error reporting $error_test_i" + if [ $KEEP_GOING -ne 0 ]; then + echo "Expect three failing commands." + fi + # If the component doesn't run in a subshell, changing error_test_i to an + # invalid integer will cause an error in the loop that runs this function. + error_test_i=this_should_not_be_used_since_the_component_runs_in_a_subshell + # Expected error: 'grep non_existent /dev/null -> 1' + grep non_existent /dev/null + # Expected error: '! grep -q . tests/scripts/all.sh -> 1' + not grep -q . "$0" + # Expected error: 'make unknown_target -> 2' + make unknown_target + false "this should not be executed" +} + +# Run one component and clean up afterwards. +run_component () { + current_component="$1" + export MBEDTLS_TEST_CONFIGURATION="$current_component" + + # Unconditionally create a seedfile that's sufficiently long. + # Do this before each component, because a previous component may + # have messed it up or shortened it. + local dd_cmd + dd_cmd=(dd if=/dev/urandom of=./tests/seedfile bs=64 count=1) + case $OSTYPE in + linux*|freebsd*|openbsd*) dd_cmd+=(status=none) + esac + "${dd_cmd[@]}" + + if [ -d tf-psa-crypto ]; then + dd_cmd=(dd if=/dev/urandom of=./tf-psa-crypto/tests/seedfile bs=64 count=1) + case $OSTYPE in + linux*|freebsd*|openbsd*) dd_cmd+=(status=none) + esac + "${dd_cmd[@]}" + fi + + # Run the component in a subshell, with error trapping and output + # redirection set up based on the relevant options. + if [ $KEEP_GOING -eq 1 ]; then + # We want to keep running if the subshell fails, so 'set -e' must + # be off when the subshell runs. + set +e + fi + ( + if [ $QUIET -eq 1 ]; then + # msg() will be silenced, so just print the component name here. + echo "${current_component#component_}" + exec >/dev/null + fi + if [ $KEEP_GOING -eq 1 ]; then + # Keep "set -e" off, and run an ERR trap instead to record failures. + set -E + trap err_trap ERR + fi + # The next line is what runs the component + "$@" + if [ $KEEP_GOING -eq 1 ]; then + trap - ERR + exit $last_failure_status + fi + ) + component_status=$? + if [ $KEEP_GOING -eq 1 ]; then + set -e + if [ $component_status -ne 0 ]; then + failure_count=$((failure_count + 1)) + fi + fi + + # Restore the build tree to a clean state. + cleanup + unset current_component +} + +################################################################ +#### Main +################################################################ + +main () { + # Preliminary setup + pre_set_shell_options + pre_set_signal_handlers + pre_check_environment + pre_load_helpers + pre_load_components + pre_initialize_variables + pre_parse_command_line "$@" + + setup_quiet_wrappers + pre_check_git + pre_restore_files + pre_back_up + + build_status=0 + if [ $KEEP_GOING -eq 1 ]; then + pre_setup_keep_going + fi + pre_prepare_outcome_file + pre_print_configuration + pre_check_tools + cleanup + if in_mbedtls_repo; then + pre_generate_files + fi + + # Run the requested tests. + for ((error_test_i=1; error_test_i <= error_test; error_test_i++)); do + run_component pseudo_component_error_test + done + unset error_test_i + for component in $RUN_COMPONENTS; do + run_component "component_$component" + done + + # We're done. + post_report +} diff --git a/tests/scripts/all-helpers.sh b/tests/scripts/all-helpers.sh new file mode 100644 index 0000000000..0e97f39190 --- /dev/null +++ b/tests/scripts/all-helpers.sh @@ -0,0 +1,267 @@ +# all-helpers.sh +# +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +# This file contains helpers for test components that are executed by all.sh. +# See "Files structure" in all-core.sh for other files used by all.sh. +# +# This file is the right place for helpers: +# - that are used by more than one component living in more than one file; +# - or (inclusive) that we want to share accross repos or branches. +# +# Helpers that are used in a single component file that is +# repo&branch-specific can be defined in the file where they are used. + +################################################################ +#### Helpers for components using libtestdriver1 +################################################################ + +# How to use libtestdriver1 +# ------------------------- +# +# 1. Define the list algorithms and key types to accelerate, +# designated the same way as PSA_WANT_ macros but without PSA_WANT_. +# Examples: +# - loc_accel_list="ALG_JPAKE" +# - loc_accel_list="ALG_FFDH KEY_TYPE_DH_KEY_PAIR KEY_TYPE_DH_PUBLIC_KEY" +# 2. Make configurations changes for the driver and/or main libraries. +# 2a. Call helper_libtestdriver1_adjust_config , where the argument +# can be either "default" to start with the default config, or a name +# supported by scripts/config.py (for example, "full"). This selects +# the base to use, and makes common adjustments. +# 2b. If desired, adjust the PSA_WANT symbols in psa/crypto_config.h. +# These changes affect both the driver and the main libraries. +# (Note: they need to have the same set of PSA_WANT symbols, as that +# determines the ABI between them.) +# 2c. Adjust MBEDTLS_ symbols in mbedtls_config.h. This only affects the +# main libraries. Typically, you want to disable the module(s) that are +# being accelerated. You may need to also disable modules that depend +# on them or options that are not supported with drivers. +# 2d. On top of psa/crypto_config.h, the driver library uses its own config +# file: tests/include/test/drivers/config_test_driver.h. You usually +# don't need to edit it: using loc_extra_list (see below) is preferred. +# However, when there's no PSA symbol for what you want to enable, +# calling scripts/config.py on this file remains the only option. +# 3. Build the driver library, then the main libraries, test, and programs. +# 3a. Call helper_libtestdriver1_make_drivers "$loc_accel_list". You may +# need to enable more algorithms here, typically hash algorithms when +# accelerating some signature algorithms (ECDSA, RSAv2). This is done +# by passing a 2nd argument listing the extra algorithms. +# Example: +# loc_extra_list="ALG_SHA_224 ALG_SHA_256 ALG_SHA_384 ALG_SHA_512" +# helper_libtestdriver1_make_drivers "$loc_accel_list" "$loc_extra_list" +# 3b. Call helper_libtestdriver1_make_main "$loc_accel_list". Any +# additional arguments will be passed to make: this can be useful if +# you don't want to build everything when iterating during development. +# Example: +# helper_libtestdriver1_make_main "$loc_accel_list" -C tests test_suite_foo +# 4. Run the tests you want. + +# Adjust the configuration - for both libtestdriver1 and main library, +# as they should have the same PSA_WANT macros. +helper_libtestdriver1_adjust_config() { + base_config=$1 + # Select the base configuration + if [ "$base_config" != "default" ]; then + scripts/config.py "$base_config" + fi + + # Enable PSA-based config (necessary to use drivers) + scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG + + # Dynamic secure element support is a deprecated feature and needs to be disabled here. + # This is done to have the same form of psa_key_attributes_s for libdriver and library. + scripts/config.py unset MBEDTLS_PSA_CRYPTO_SE_C + + # If threading is enabled on the normal build, then we need to enable it in the drivers as well, + # otherwise we will end up running multithreaded tests without mutexes to protect them. + if scripts/config.py get MBEDTLS_THREADING_C; then + scripts/config.py -f "$CONFIG_TEST_DRIVER_H" set MBEDTLS_THREADING_C + fi + + if scripts/config.py get MBEDTLS_THREADING_PTHREAD; then + scripts/config.py -f "$CONFIG_TEST_DRIVER_H" set MBEDTLS_THREADING_PTHREAD + fi +} + +# Build the drivers library libtestdriver1.a (with ASan). +# +# Parameters: +# 1. a space-separated list of things to accelerate; +# 2. optional: a space-separate list of things to also support. +# Here "things" are PSA_WANT_ symbols but with PSA_WANT_ removed. +helper_libtestdriver1_make_drivers() { + loc_accel_flags=$( echo "$1 ${2-}" | sed 's/[^ ]* */-DLIBTESTDRIVER1_MBEDTLS_PSA_ACCEL_&/g' ) + make CC=$ASAN_CC -C tests libtestdriver1.a CFLAGS=" $ASAN_CFLAGS $loc_accel_flags" LDFLAGS="$ASAN_CFLAGS" +} + +# Build the main libraries, programs and tests, +# linking to the drivers library (with ASan). +# +# Parameters: +# 1. a space-separated list of things to accelerate; +# *. remaining arguments if any are passed directly to make +# (examples: lib, -C tests test_suite_xxx, etc.) +# Here "things" are PSA_WANT_ symbols but with PSA_WANT_ removed. +helper_libtestdriver1_make_main() { + loc_accel_list=$1 + shift + + # we need flags both with and without the LIBTESTDRIVER1_ prefix + loc_accel_flags=$( echo "$loc_accel_list" | sed 's/[^ ]* */-DLIBTESTDRIVER1_MBEDTLS_PSA_ACCEL_&/g' ) + loc_accel_flags="$loc_accel_flags $( echo "$loc_accel_list" | sed 's/[^ ]* */-DMBEDTLS_PSA_ACCEL_&/g' )" + make CC=$ASAN_CC CFLAGS="$ASAN_CFLAGS -I../tests/include -I../tests -I../../tests -DPSA_CRYPTO_DRIVER_TEST -DMBEDTLS_TEST_LIBTESTDRIVER1 $loc_accel_flags" LDFLAGS="-ltestdriver1 $ASAN_CFLAGS" "$@" +} + +################################################################ +#### Helpers for components using psasim +################################################################ + +# Set some default values $CONFIG_H in order to build server or client sides +# in PSASIM. There is only 1 mandatory parameter: +# - $1: target which can be "client" or "server" +helper_psasim_config() { + TARGET=$1 + + if [ "$TARGET" == "client" ]; then + scripts/config.py full + scripts/config.py unset MBEDTLS_PSA_CRYPTO_C + scripts/config.py unset MBEDTLS_PSA_CRYPTO_STORAGE_C + # Dynamic secure element support is a deprecated feature and it is not + # available when CRYPTO_C and PSA_CRYPTO_STORAGE_C are disabled. + scripts/config.py unset MBEDTLS_PSA_CRYPTO_SE_C + # Disable potentially problematic features + scripts/config.py unset MBEDTLS_X509_RSASSA_PSS_SUPPORT + scripts/config.py unset MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED + scripts/config.py unset MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED + scripts/config.py unset MBEDTLS_KEY_EXCHANGE_ECDH_RSA_ENABLED + scripts/config.py unset MBEDTLS_ECP_RESTARTABLE + scripts/config.py unset MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER + else + scripts/config.py crypto_full + scripts/config.py unset MBEDTLS_PSA_CRYPTO_BUILTIN_KEYS + # We need to match the client with MBEDTLS_PSA_CRYPTO_SE_C + scripts/config.py unset MBEDTLS_PSA_CRYPTO_SE_C + # Also ensure MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER not set (to match client) + scripts/config.py unset MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER + fi +} + +# This is a helper function to be used in psasim builds. It is meant to clean +# up the library's workspace after the server build and before the client +# build. Built libraries (mbedcrypto, mbedx509 and mbedtls) are supposed to be +# already copied to psasim folder at this point. +helper_psasim_cleanup_before_client() { + # Clean up library files + make -C library clean + + # Restore files that were backup before building library files. This + # includes $CONFIG_H and $CRYPTO_CONFIG_H. + restore_backed_up_files +} + +# Helper to build the libraries for client/server in PSASIM. If the server is +# being built, then it builds also the final executable. +# There is only 1 mandatory parameter: +# - $1: target which can be "client" or "server" +helper_psasim_build() { + TARGET=$1 + shift + TARGET_LIB=${TARGET}_libs + + make -C $PSASIM_PATH CFLAGS="$ASAN_CFLAGS" LDFLAGS="$ASAN_CFLAGS" $TARGET_LIB "$@" + + # Build also the server application after its libraries have been built. + if [ "$TARGET" == "server" ]; then + make -C $PSASIM_PATH CFLAGS="$ASAN_CFLAGS" LDFLAGS="$ASAN_CFLAGS" test/psa_server + fi +} + +################################################################ +#### Configuration helpers +################################################################ + +# When called with no parameter this function disables all builtin curves. +# The function optionally accepts 1 parameter: a space-separated list of the +# curves that should be kept enabled. +helper_disable_builtin_curves() { + allowed_list="${1:-}" + scripts/config.py unset-all "MBEDTLS_ECP_DP_[0-9A-Z_a-z]*_ENABLED" + + for curve in $allowed_list; do + scripts/config.py set $curve + done +} + +# Helper returning the list of supported elliptic curves from CRYPTO_CONFIG_H, +# without the "PSA_WANT_" prefix. This becomes handy for accelerating curves +# in the following helpers. +helper_get_psa_curve_list () { + loc_list="" + for item in $(sed -n 's/^#define PSA_WANT_\(ECC_[0-9A-Z_a-z]*\).*/\1/p' <"$CRYPTO_CONFIG_H"); do + loc_list="$loc_list $item" + done + + echo "$loc_list" +} + +# Helper returning the list of supported DH groups from CRYPTO_CONFIG_H, +# without the "PSA_WANT_" prefix. This becomes handy for accelerating DH groups +# in the following helpers. +helper_get_psa_dh_group_list () { + loc_list="" + for item in $(sed -n 's/^#define PSA_WANT_\(DH_RFC7919_[0-9]*\).*/\1/p' <"$CRYPTO_CONFIG_H"); do + loc_list="$loc_list $item" + done + + echo "$loc_list" +} + +# Get the list of uncommented PSA_WANT_KEY_TYPE_xxx_ from CRYPTO_CONFIG_H. This +# is useful to easily get a list of key type symbols to accelerate. +# The function accepts a single argument which is the key type: ECC, DH, RSA. +helper_get_psa_key_type_list() { + key_type="$1" + loc_list="" + for item in $(sed -n "s/^#define PSA_WANT_\(KEY_TYPE_${key_type}_[0-9A-Z_a-z]*\).*/\1/p" <"$CRYPTO_CONFIG_H"); do + # Skip DERIVE for elliptic keys since there is no driver dispatch for + # it so it cannot be accelerated. + if [ "$item" != "KEY_TYPE_ECC_KEY_PAIR_DERIVE" ]; then + loc_list="$loc_list $item" + fi + done + + echo "$loc_list" +} + +################################################################ +#### Misc. helpers for components +################################################################ + +helper_armc6_build_test() +{ + FLAGS="$1" + + msg "build: ARM Compiler 6 ($FLAGS)" + make clean + ARM_TOOL_VARIANT="ult" CC="$ARMC6_CC" AR="$ARMC6_AR" CFLAGS="$FLAGS" \ + WARNING_CFLAGS='-Werror -xc -std=c99' make lib + + msg "size: ARM Compiler 6 ($FLAGS)" + "$ARMC6_FROMELF" -z library/*.o + if [ -n "${PSA_CORE_PATH}" ]; then + "$ARMC6_FROMELF" -z ${PSA_CORE_PATH}/*.o + fi + if [ -n "${BUILTIN_SRC_PATH}" ]; then + "$ARMC6_FROMELF" -z ${BUILTIN_SRC_PATH}/*.o + fi +} + +clang_version() { + if command -v clang > /dev/null ; then + clang --version|grep version|sed -E 's#.*version ([0-9]+).*#\1#' + else + echo 0 # report version 0 for "no clang" + fi +} diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 1a73020812..6708de1027 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -5,1130 +5,11 @@ # Copyright The Mbed TLS Contributors # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# This file is executable; it is the entry point for users and the CI. +# See "Files structure" in all-core.sh for other files used. - -################################################################ -#### Documentation -################################################################ - -# Purpose -# ------- -# -# To run all tests possible or available on the platform. -# -# Notes for users -# --------------- -# -# Warning: the test is destructive. It includes various build modes and -# configurations, and can and will arbitrarily change the current CMake -# configuration. The following files must be committed into git: -# * include/mbedtls/mbedtls_config.h -# * Makefile, library/Makefile, programs/Makefile, tests/Makefile, -# programs/fuzz/Makefile -# After running this script, the CMake cache will be lost and CMake -# will no longer be initialised. -# -# The script assumes the presence of a number of tools: -# * Basic Unix tools (Windows users note: a Unix-style find must be before -# the Windows find in the PATH) -# * Perl -# * GNU Make -# * CMake -# * GCC and Clang (recent enough for using ASan with gcc and MemSan with clang, or valgrind) -# * G++ -# * arm-gcc and mingw-gcc -# * ArmCC 5 and ArmCC 6, unless invoked with --no-armcc -# * OpenSSL and GnuTLS command line tools, in suitable versions for the -# interoperability tests. The following are the official versions at the -# time of writing: -# * GNUTLS_{CLI,SERV} = 3.4.10 -# * GNUTLS_NEXT_{CLI,SERV} = 3.7.2 -# * OPENSSL = 1.0.2g (without Debian/Ubuntu patches) -# * OPENSSL_NEXT = 3.1.2 -# See the invocation of check_tools below for details. -# -# This script must be invoked from the toplevel directory of a git -# working copy of Mbed TLS. -# -# The behavior on an error depends on whether --keep-going (alias -k) -# is in effect. -# * Without --keep-going: the script stops on the first error without -# cleaning up. This lets you work in the configuration of the failing -# component. -# * With --keep-going: the script runs all requested components and -# reports failures at the end. In particular the script always cleans -# up on exit. -# -# Note that the output is not saved. You may want to run -# script -c tests/scripts/all.sh -# or -# tests/scripts/all.sh >all.log 2>&1 -# -# Notes for maintainers -# --------------------- -# -# The bulk of the code is organized into functions that follow one of the -# following naming conventions: -# * pre_XXX: things to do before running the tests, in order. -# * component_XXX: independent components. They can be run in any order. -# * component_check_XXX: quick tests that aren't worth parallelizing. -# * component_build_XXX: build things but don't run them. -# * component_test_XXX: build and test. -# * component_release_XXX: tests that the CI should skip during PR testing. -# * support_XXX: if support_XXX exists and returns false then -# component_XXX is not run by default. -# * post_XXX: things to do after running the tests. -# * other: miscellaneous support functions. -# -# Each component must start by invoking `msg` with a short informative message. -# -# Warning: due to the way bash detects errors, the failure of a command -# inside 'if' or '!' is not detected. Use the 'not' function instead of '!'. -# -# Each component is executed in a separate shell process. The component -# fails if any command in it returns a non-zero status. -# -# The framework performs some cleanup tasks after each component. This -# means that components can assume that the working directory is in a -# cleaned-up state, and don't need to perform the cleanup themselves. -# * Run `make clean`. -# * Restore `include/mbedtls/mbedtls_config.h` from a backup made before running -# the component. -# * Check out `Makefile`, `library/Makefile`, `programs/Makefile`, -# `tests/Makefile` and `programs/fuzz/Makefile` from git. -# This cleans up after an in-tree use of CMake. -# -# The tests are roughly in order from fastest to slowest. This doesn't -# have to be exact, but in general you should add slower tests towards -# the end and fast checks near the beginning. - - - -################################################################ -#### Initialization and command line parsing -################################################################ - -# Abort on errors (even on the left-hand side of a pipe). -# Treat uninitialised variables as errors. -set -e -o pipefail -u - -# Enable ksh/bash extended file matching patterns -shopt -s extglob - -# For project detection -in_mbedtls_repo () { - test "$PROJECT_NAME" = "Mbed TLS" -} - -in_tf_psa_crypto_repo () { - test "$PROJECT_NAME" = "TF-PSA-Crypto" -} - -pre_check_environment () { - # For project detection - PROJECT_NAME_FILE='./scripts/project_name.txt' - if read -r PROJECT_NAME < "$PROJECT_NAME_FILE"; then :; else - echo "$PROJECT_NAME_FILE does not exist... Exiting..." >&2 - exit 1 - fi - - if in_mbedtls_repo || in_tf_psa_crypto_repo; then :; else - echo "Must be run from Mbed TLS / TF-PSA-Crypto root" >&2 - exit 1 - fi -} - -pre_initialize_variables () { - if in_mbedtls_repo; then - CONFIG_H='include/mbedtls/mbedtls_config.h' - else - CONFIG_H='drivers/builtin/include/mbedtls/mbedtls_config.h' - fi - CRYPTO_CONFIG_H='include/psa/crypto_config.h' - CONFIG_TEST_DRIVER_H='tests/include/test/drivers/config_test_driver.h' - - # Files that are clobbered by some jobs will be backed up. Use a different - # suffix from auxiliary scripts so that all.sh and auxiliary scripts can - # independently decide when to remove the backup file. - backup_suffix='.all.bak' - # Files clobbered by config.py - files_to_back_up="$CONFIG_H $CRYPTO_CONFIG_H $CONFIG_TEST_DRIVER_H" - if in_mbedtls_repo; then - # Files clobbered by in-tree cmake - files_to_back_up="$files_to_back_up Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile" - fi - - append_outcome=0 - MEMORY=0 - FORCE=0 - QUIET=0 - KEEP_GOING=0 - - # Seed value used with the --release-test option. - # - # See also RELEASE_SEED in basic-build-test.sh. Debugging is easier if - # both values are kept in sync. If you change the value here because it - # breaks some tests, you'll definitely want to change it in - # basic-build-test.sh as well. - RELEASE_SEED=1 - - # Specify character collation for regular expressions and sorting with C locale - export LC_COLLATE=C - - : ${MBEDTLS_TEST_OUTCOME_FILE=} - : ${MBEDTLS_TEST_PLATFORM="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"} - export MBEDTLS_TEST_OUTCOME_FILE - export MBEDTLS_TEST_PLATFORM - - # Default commands, can be overridden by the environment - : ${OPENSSL:="openssl"} - : ${OPENSSL_NEXT:="$OPENSSL"} - : ${GNUTLS_CLI:="gnutls-cli"} - : ${GNUTLS_SERV:="gnutls-serv"} - : ${OUT_OF_SOURCE_DIR:=./mbedtls_out_of_source_build} - : ${ARMC5_BIN_DIR:=/usr/bin} - : ${ARMC6_BIN_DIR:=/usr/bin} - : ${ARM_NONE_EABI_GCC_PREFIX:=arm-none-eabi-} - : ${ARM_LINUX_GNUEABI_GCC_PREFIX:=arm-linux-gnueabi-} - : ${CLANG_LATEST:="clang-latest"} - : ${CLANG_EARLIEST:="clang-earliest"} - : ${GCC_LATEST:="gcc-latest"} - : ${GCC_EARLIEST:="gcc-earliest"} - # if MAKEFLAGS is not set add the -j option to speed up invocations of make - if [ -z "${MAKEFLAGS+set}" ]; then - export MAKEFLAGS="-j$(all_sh_nproc)" - fi - # if CC is not set, use clang by default (if present) to improve build times - if [ -z "${CC+set}" ] && (type clang > /dev/null 2>&1); then - export CC="clang" - fi - - if [ -n "${OPENSSL_3+set}" ]; then - export OPENSSL_NEXT="$OPENSSL_3" - fi - - # Include more verbose output for failing tests run by CMake or make - export CTEST_OUTPUT_ON_FAILURE=1 - - # CFLAGS and LDFLAGS for Asan builds that don't use CMake - # default to -O2, use -Ox _after_ this if you want another level - ASAN_CFLAGS='-O2 -Werror -fsanitize=address,undefined -fno-sanitize-recover=all' - # Normally, tests should use this compiler for ASAN testing - ASAN_CC=clang - - # Platform tests have an allocation that returns null - export ASAN_OPTIONS="allocator_may_return_null=1" - export MSAN_OPTIONS="allocator_may_return_null=1" - - # Gather the list of available components. These are the functions - # defined in this script whose name starts with "component_". - ALL_COMPONENTS=$(compgen -A function component_ | sed 's/component_//') - - # Delay determining SUPPORTED_COMPONENTS until the command line options have a chance to override - # the commands set by the environment -} - -setup_quiet_wrappers() -{ - # Pick up "quiet" wrappers for make and cmake, which don't output very much - # unless there is an error. This reduces logging overhead in the CI. - # - # Note that the cmake wrapper breaks unless we use an absolute path here. - if [[ -e ${PWD}/tests/scripts/quiet ]]; then - export PATH=${PWD}/tests/scripts/quiet:$PATH - fi -} - -# Test whether the component $1 is included in the command line patterns. -is_component_included() -{ - # Temporarily disable wildcard expansion so that $COMMAND_LINE_COMPONENTS - # only does word splitting. - set -f - for pattern in $COMMAND_LINE_COMPONENTS; do - set +f - case ${1#component_} in $pattern) return 0;; esac - done - set +f - return 1 -} - -usage() -{ - cat < - Prefix for a cross-compiler for arm-none-eabi - (default: "${ARM_NONE_EABI_GCC_PREFIX}") - --arm-linux-gnueabi-gcc-prefix= - Prefix for a cross-compiler for arm-linux-gnueabi - (default: "${ARM_LINUX_GNUEABI_GCC_PREFIX}") - --armcc Run ARM Compiler builds (on by default). - --restore First clean up the build tree, restoring backed up - files. Do not run any components unless they are - explicitly specified. - --error-test Error test mode: run a failing function in addition - to any specified component. May be repeated. - --except Exclude the COMPONENTs listed on the command line, - instead of running only those. - --no-append-outcome Write a new outcome file and analyze it (default). - --no-armcc Skip ARM Compiler builds. - --no-force Refuse to overwrite modified files (default). - --no-keep-going Stop at the first error (default). - --no-memory No additional memory tests (default). - --no-quiet Print full output from components. - --out-of-source-dir= Directory used for CMake out-of-source build tests. - --outcome-file= File where test outcomes are written (not done if - empty; default: \$MBEDTLS_TEST_OUTCOME_FILE). - --random-seed Use a random seed value for randomized tests (default). - -r|--release-test Run this script in release mode. This fixes the seed value to ${RELEASE_SEED}. - -s|--seed Integer seed value to use for this test run. - -Tool path options: - --armc5-bin-dir= ARM Compiler 5 bin directory. - --armc6-bin-dir= ARM Compiler 6 bin directory. - --clang-earliest= Earliest version of clang available - --clang-latest= Latest version of clang available - --gcc-earliest= Earliest version of GCC available - --gcc-latest= Latest version of GCC available - --gnutls-cli= GnuTLS client executable to use for most tests. - --gnutls-serv= GnuTLS server executable to use for most tests. - --openssl= OpenSSL executable to use for most tests. - --openssl-next= OpenSSL executable to use for recent things like ARIA -EOF -} - -# Cleanup before/after running a component. -# Remove built files as well as the cmake cache/config. -# Does not remove generated source files. -cleanup() -{ - if in_mbedtls_repo; then - command make clean - fi - - # Remove CMake artefacts - find . -name .git -prune -o \ - -iname CMakeFiles -exec rm -rf {} \+ -o \ - \( -iname cmake_install.cmake -o \ - -iname CTestTestfile.cmake -o \ - -iname CMakeCache.txt -o \ - -path './cmake/*.cmake' \) -exec rm -f {} \+ - # Remove Makefiles generated by in-tree CMake builds - rm -f 3rdparty/Makefile 3rdparty/*/Makefile pkgconfig/Makefile framework/Makefile - rm -f include/Makefile programs/!(fuzz)/Makefile - - # Remove any artifacts from the component_test_cmake_as_subdirectory test. - rm -rf programs/test/cmake_subproject/build - rm -f programs/test/cmake_subproject/Makefile - rm -f programs/test/cmake_subproject/cmake_subproject - - # Remove any artifacts from the component_test_cmake_as_package test. - rm -rf programs/test/cmake_package/build - rm -f programs/test/cmake_package/Makefile - rm -f programs/test/cmake_package/cmake_package - - # Remove any artifacts from the component_test_cmake_as_installed_package test. - rm -rf programs/test/cmake_package_install/build - rm -f programs/test/cmake_package_install/Makefile - rm -f programs/test/cmake_package_install/cmake_package_install - - # Restore files that may have been clobbered by the job - for x in $files_to_back_up; do - if [[ -e "$x$backup_suffix" ]]; then - cp -p "$x$backup_suffix" "$x" - fi - done -} - -# Final cleanup when this script exits (except when exiting on a failure -# in non-keep-going mode). -final_cleanup () { - cleanup - - for x in $files_to_back_up; do - rm -f "$x$backup_suffix" - done -} - -# Executed on exit. May be redefined depending on command line options. -final_report () { - : -} - -fatal_signal () { - final_cleanup - final_report $1 - trap - $1 - kill -$1 $$ -} - -trap 'fatal_signal HUP' HUP -trap 'fatal_signal INT' INT -trap 'fatal_signal TERM' TERM - -# Number of processors on this machine. Used as the default setting -# for parallel make. -all_sh_nproc () -{ - { - nproc || # Linux - sysctl -n hw.ncpuonline || # NetBSD, OpenBSD - sysctl -n hw.ncpu || # FreeBSD - echo 1 - } 2>/dev/null -} - -msg() -{ - if [ -n "${current_component:-}" ]; then - current_section="${current_component#component_}: $1" - else - current_section="$1" - fi - - if [ $QUIET -eq 1 ]; then - return - fi - - echo "" - echo "******************************************************************" - echo "* $current_section " - printf "* "; date - echo "******************************************************************" -} - -armc6_build_test() -{ - FLAGS="$1" - - msg "build: ARM Compiler 6 ($FLAGS)" - make clean - ARM_TOOL_VARIANT="ult" CC="$ARMC6_CC" AR="$ARMC6_AR" CFLAGS="$FLAGS" \ - WARNING_CFLAGS='-Werror -xc -std=c99' make lib - - msg "size: ARM Compiler 6 ($FLAGS)" - "$ARMC6_FROMELF" -z library/*.o -} - -err_msg() -{ - echo "$1" >&2 -} - -check_tools() -{ - for tool in "$@"; do - if ! `type "$tool" >/dev/null 2>&1`; then - err_msg "$tool not found!" - exit 1 - fi - done -} - -pre_parse_command_line () { - COMMAND_LINE_COMPONENTS= - all_except=0 - error_test=0 - list_components=0 - restore_first=0 - no_armcc= - - # Note that legacy options are ignored instead of being omitted from this - # list of options, so invocations that worked with previous version of - # all.sh will still run and work properly. - while [ $# -gt 0 ]; do - case "$1" in - --append-outcome) append_outcome=1;; - --arm-none-eabi-gcc-prefix) shift; ARM_NONE_EABI_GCC_PREFIX="$1";; - --arm-linux-gnueabi-gcc-prefix) shift; ARM_LINUX_GNUEABI_GCC_PREFIX="$1";; - --armcc) no_armcc=;; - --armc5-bin-dir) shift; ARMC5_BIN_DIR="$1";; - --armc6-bin-dir) shift; ARMC6_BIN_DIR="$1";; - --clang-earliest) shift; CLANG_EARLIEST="$1";; - --clang-latest) shift; CLANG_LATEST="$1";; - --error-test) error_test=$((error_test + 1));; - --except) all_except=1;; - --force|-f) FORCE=1;; - --gcc-earliest) shift; GCC_EARLIEST="$1";; - --gcc-latest) shift; GCC_LATEST="$1";; - --gnutls-cli) shift; GNUTLS_CLI="$1";; - --gnutls-legacy-cli) shift;; # ignored for backward compatibility - --gnutls-legacy-serv) shift;; # ignored for backward compatibility - --gnutls-serv) shift; GNUTLS_SERV="$1";; - --help|-h) usage; exit;; - --keep-going|-k) KEEP_GOING=1;; - --list-all-components) printf '%s\n' $ALL_COMPONENTS; exit;; - --list-components) list_components=1;; - --memory|-m) MEMORY=1;; - --no-append-outcome) append_outcome=0;; - --no-armcc) no_armcc=1;; - --no-force) FORCE=0;; - --no-keep-going) KEEP_GOING=0;; - --no-memory) MEMORY=0;; - --no-quiet) QUIET=0;; - --openssl) shift; OPENSSL="$1";; - --openssl-next) shift; OPENSSL_NEXT="$1";; - --outcome-file) shift; MBEDTLS_TEST_OUTCOME_FILE="$1";; - --out-of-source-dir) shift; OUT_OF_SOURCE_DIR="$1";; - --quiet|-q) QUIET=1;; - --random-seed) unset SEED;; - --release-test|-r) SEED=$RELEASE_SEED;; - --restore) restore_first=1;; - --seed|-s) shift; SEED="$1";; - -*) - echo >&2 "Unknown option: $1" - echo >&2 "Run $0 --help for usage." - exit 120 - ;; - *) COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS $1";; - esac - shift - done - - # Exclude components that are not supported on this platform. - SUPPORTED_COMPONENTS= - for component in $ALL_COMPONENTS; do - case $(type "support_$component" 2>&1) in - *' function'*) - if ! support_$component; then continue; fi;; - esac - SUPPORTED_COMPONENTS="$SUPPORTED_COMPONENTS $component" - done - - if [ $list_components -eq 1 ]; then - printf '%s\n' $SUPPORTED_COMPONENTS - exit - fi - - # With no list of components, run everything. - if [ -z "$COMMAND_LINE_COMPONENTS" ] && [ $restore_first -eq 0 ]; then - all_except=1 - fi - - # --no-armcc is a legacy option. The modern way is --except '*_armcc*'. - # Ignore it if components are listed explicitly on the command line. - if [ -n "$no_armcc" ] && [ $all_except -eq 1 ]; then - COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS *_armcc*" - fi - - # Error out if an explicitly requested component doesn't exist. - if [ $all_except -eq 0 ]; then - unsupported=0 - # Temporarily disable wildcard expansion so that $COMMAND_LINE_COMPONENTS - # only does word splitting. - set -f - for component in $COMMAND_LINE_COMPONENTS; do - set +f - # If the requested name includes a wildcard character, don't - # check it. Accept wildcard patterns that don't match anything. - case $component in - *[*?\[]*) continue;; - esac - case " $SUPPORTED_COMPONENTS " in - *" $component "*) :;; - *) - echo >&2 "Component $component was explicitly requested, but is not known or not supported." - unsupported=$((unsupported + 1));; - esac - done - set +f - if [ $unsupported -ne 0 ]; then - exit 2 - fi - fi - - # Build the list of components to run. - RUN_COMPONENTS= - for component in $SUPPORTED_COMPONENTS; do - if is_component_included "$component"; [ $? -eq $all_except ]; then - RUN_COMPONENTS="$RUN_COMPONENTS $component" - fi - done - - unset all_except - unset no_armcc -} - -pre_check_git () { - if [ $FORCE -eq 1 ]; then - rm -rf "$OUT_OF_SOURCE_DIR" - git checkout-index -f -q $CONFIG_H - cleanup - else - - if [ -d "$OUT_OF_SOURCE_DIR" ]; then - echo "Warning - there is an existing directory at '$OUT_OF_SOURCE_DIR'" >&2 - echo "You can either delete this directory manually, or force the test by rerunning" - echo "the script as: $0 --force --out-of-source-dir $OUT_OF_SOURCE_DIR" - exit 1 - fi - - if ! git diff --quiet "$CONFIG_H"; then - err_msg "Warning - the configuration file '$CONFIG_H' has been edited. " - echo "You can either delete or preserve your work, or force the test by rerunning the" - echo "script as: $0 --force" - exit 1 - fi - fi -} - -pre_restore_files () { - # If the makefiles have been generated by a framework such as cmake, - # restore them from git. If the makefiles look like modifications from - # the ones checked into git, take care not to modify them. Whatever - # this function leaves behind is what the script will restore before - # each component. - case "$(head -n1 Makefile)" in - *[Gg]enerated*) - git update-index --no-skip-worktree Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile - git checkout -- Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile - ;; - esac -} - -pre_back_up () { - for x in $files_to_back_up; do - cp -p "$x" "$x$backup_suffix" - done -} - -pre_setup_keep_going () { - failure_count=0 # Number of failed components - last_failure_status=0 # Last failure status in this component - - # See err_trap - previous_failure_status=0 - previous_failed_command= - previous_failure_funcall_depth=0 - unset report_failed_command - - start_red= - end_color= - if [ -t 1 ]; then - case "${TERM:-}" in - *color*|cygwin|linux|rxvt*|screen|[Eex]term*) - start_red=$(printf '\033[31m') - end_color=$(printf '\033[0m') - ;; - esac - fi - - # Keep a summary of failures in a file. We'll print it out at the end. - failure_summary_file=$PWD/all-sh-failures-$$.log - : >"$failure_summary_file" - - # Whether it makes sense to keep a component going after the specified - # command fails (test command) or not (configure or build). - # This function normally receives the failing simple command - # ($BASH_COMMAND) as an argument, but if $report_failed_command is set, - # this is passed instead. - # This doesn't have to be 100% accurate: all failures are recorded anyway. - # False positives result in running things that can't be expected to - # work. False negatives result in things not running after something else - # failed even though they might have given useful feedback. - can_keep_going_after_failure () { - case "$1" in - "msg "*) false;; - "cd "*) false;; - "diff "*) true;; - *make*[\ /]tests*) false;; # make tests, make CFLAGS=-I../tests, ... - *test*) true;; # make test, tests/stuff, env V=v tests/stuff, ... - *make*check*) true;; - "grep "*) true;; - "[ "*) true;; - "! "*) true;; - *) false;; - esac - } - - # This function runs if there is any error in a component. - # It must either exit with a nonzero status, or set - # last_failure_status to a nonzero value. - err_trap () { - # Save $? (status of the failing command). This must be the very - # first thing, before $? is overridden. - last_failure_status=$? - failed_command=${report_failed_command-$BASH_COMMAND} - - if [[ $last_failure_status -eq $previous_failure_status && - "$failed_command" == "$previous_failed_command" && - ${#FUNCNAME[@]} == $((previous_failure_funcall_depth - 1)) ]] - then - # The same command failed twice in a row, but this time one level - # less deep in the function call stack. This happens when the last - # command of a function returns a nonzero status, and the function - # returns that same status. Ignore the second failure. - previous_failure_funcall_depth=${#FUNCNAME[@]} - return - fi - previous_failure_status=$last_failure_status - previous_failed_command=$failed_command - previous_failure_funcall_depth=${#FUNCNAME[@]} - - text="$current_section: $failed_command -> $last_failure_status" - echo "${start_red}^^^^$text^^^^${end_color}" >&2 - echo "$text" >>"$failure_summary_file" - - # If the command is fatal (configure or build command), stop this - # component. Otherwise (test command) keep the component running - # (run more tests from the same build). - if ! can_keep_going_after_failure "$failed_command"; then - exit $last_failure_status - fi - } - - final_report () { - if [ $failure_count -gt 0 ]; then - echo - echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" - echo "${start_red}FAILED: $failure_count components${end_color}" - cat "$failure_summary_file" - echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" - elif [ -z "${1-}" ]; then - echo "SUCCESS :)" - fi - if [ -n "${1-}" ]; then - echo "Killed by SIG$1." - fi - rm -f "$failure_summary_file" - if [ $failure_count -gt 0 ]; then - exit 1 - fi - } -} - -# record_status() and if_build_succeeded() are kept temporarily for backward -# compatibility. Don't use them in new components. -record_status () { - "$@" -} -if_build_succeeded () { - "$@" -} - -# '! true' does not trigger the ERR trap. Arrange to trigger it, with -# a reasonably informative error message (not just "$@"). -not () { - if "$@"; then - report_failed_command="! $*" - false - unset report_failed_command - fi -} - -pre_prepare_outcome_file () { - case "$MBEDTLS_TEST_OUTCOME_FILE" in - [!/]*) MBEDTLS_TEST_OUTCOME_FILE="$PWD/$MBEDTLS_TEST_OUTCOME_FILE";; - esac - if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ] && [ "$append_outcome" -eq 0 ]; then - rm -f "$MBEDTLS_TEST_OUTCOME_FILE" - fi -} - -pre_print_configuration () { - if [ $QUIET -eq 1 ]; then - return - fi - - msg "info: $0 configuration" - echo "MEMORY: $MEMORY" - echo "FORCE: $FORCE" - echo "MBEDTLS_TEST_OUTCOME_FILE: ${MBEDTLS_TEST_OUTCOME_FILE:-(none)}" - echo "SEED: ${SEED-"UNSET"}" - echo - echo "OPENSSL: $OPENSSL" - echo "OPENSSL_NEXT: $OPENSSL_NEXT" - echo "GNUTLS_CLI: $GNUTLS_CLI" - echo "GNUTLS_SERV: $GNUTLS_SERV" - echo "ARMC5_BIN_DIR: $ARMC5_BIN_DIR" - echo "ARMC6_BIN_DIR: $ARMC6_BIN_DIR" -} - -# Make sure the tools we need are available. -pre_check_tools () { - # Build the list of variables to pass to output_env.sh. - set env - - case " $RUN_COMPONENTS " in - # Require OpenSSL and GnuTLS if running any tests (as opposed to - # only doing builds). Not all tests run OpenSSL and GnuTLS, but this - # is a good enough approximation in practice. - *" test_"* | *" release_test_"*) - # To avoid setting OpenSSL and GnuTLS for each call to compat.sh - # and ssl-opt.sh, we just export the variables they require. - export OPENSSL="$OPENSSL" - export GNUTLS_CLI="$GNUTLS_CLI" - export GNUTLS_SERV="$GNUTLS_SERV" - # Avoid passing --seed flag in every call to ssl-opt.sh - if [ -n "${SEED-}" ]; then - export SEED - fi - set "$@" OPENSSL="$OPENSSL" - set "$@" GNUTLS_CLI="$GNUTLS_CLI" GNUTLS_SERV="$GNUTLS_SERV" - check_tools "$OPENSSL" "$OPENSSL_NEXT" \ - "$GNUTLS_CLI" "$GNUTLS_SERV" - ;; - esac - - case " $RUN_COMPONENTS " in - *_doxygen[_\ ]*) check_tools "doxygen" "dot";; - esac - - case " $RUN_COMPONENTS " in - *_arm_none_eabi_gcc[_\ ]*) check_tools "${ARM_NONE_EABI_GCC_PREFIX}gcc";; - esac - - case " $RUN_COMPONENTS " in - *_mingw[_\ ]*) check_tools "i686-w64-mingw32-gcc";; - esac - - case " $RUN_COMPONENTS " in - *" test_zeroize "*) check_tools "gdb";; - esac - - case " $RUN_COMPONENTS " in - *_armcc*) - ARMC5_CC="$ARMC5_BIN_DIR/armcc" - ARMC5_AR="$ARMC5_BIN_DIR/armar" - ARMC5_FROMELF="$ARMC5_BIN_DIR/fromelf" - ARMC6_CC="$ARMC6_BIN_DIR/armclang" - ARMC6_AR="$ARMC6_BIN_DIR/armar" - ARMC6_FROMELF="$ARMC6_BIN_DIR/fromelf" - check_tools "$ARMC5_CC" "$ARMC5_AR" "$ARMC5_FROMELF" \ - "$ARMC6_CC" "$ARMC6_AR" "$ARMC6_FROMELF";; - esac - - # past this point, no call to check_tool, only printing output - if [ $QUIET -eq 1 ]; then - return - fi - - msg "info: output_env.sh" - case $RUN_COMPONENTS in - *_armcc*) - set "$@" ARMC5_CC="$ARMC5_CC" ARMC6_CC="$ARMC6_CC" RUN_ARMCC=1;; - *) set "$@" RUN_ARMCC=0;; - esac - "$@" scripts/output_env.sh -} - -pre_generate_files () { - # since make doesn't have proper dependencies, remove any possibly outdate - # file that might be around before generating fresh ones - make neat - if [ $QUIET -eq 1 ]; then - make generated_files >/dev/null - else - make generated_files - fi -} - -clang_version () { - if command -v clang > /dev/null ; then - clang --version|grep version|sed -E 's#.*version ([0-9]+).*#\1#' - else - echo 0 # report version 0 for "no clang" - fi -} - -################################################################ -#### Helpers for components using libtestdriver1 -################################################################ - -# How to use libtestdriver1 -# ------------------------- -# -# 1. Define the list algorithms and key types to accelerate, -# designated the same way as PSA_WANT_ macros but without PSA_WANT_. -# Examples: -# - loc_accel_list="ALG_JPAKE" -# - loc_accel_list="ALG_FFDH KEY_TYPE_DH_KEY_PAIR KEY_TYPE_DH_PUBLIC_KEY" -# 2. Make configurations changes for the driver and/or main libraries. -# 2a. Call helper_libtestdriver1_adjust_config , where the argument -# can be either "default" to start with the default config, or a name -# supported by scripts/config.py (for example, "full"). This selects -# the base to use, and makes common adjustments. -# 2b. If desired, adjust the PSA_WANT symbols in psa/crypto_config.h. -# These changes affect both the driver and the main libraries. -# (Note: they need to have the same set of PSA_WANT symbols, as that -# determines the ABI between them.) -# 2c. Adjust MBEDTLS_ symbols in mbedtls_config.h. This only affects the -# main libraries. Typically, you want to disable the module(s) that are -# being accelerated. You may need to also disable modules that depend -# on them or options that are not supported with drivers. -# 2d. On top of psa/crypto_config.h, the driver library uses its own config -# file: tests/include/test/drivers/config_test_driver.h. You usually -# don't need to edit it: using loc_extra_list (see below) is preferred. -# However, when there's no PSA symbol for what you want to enable, -# calling scripts/config.py on this file remains the only option. -# 3. Build the driver library, then the main libraries, test, and programs. -# 3a. Call helper_libtestdriver1_make_drivers "$loc_accel_list". You may -# need to enable more algorithms here, typically hash algorithms when -# accelerating some signature algorithms (ECDSA, RSAv2). This is done -# by passing a 2nd argument listing the extra algorithms. -# Example: -# loc_extra_list="ALG_SHA_224 ALG_SHA_256 ALG_SHA_384 ALG_SHA_512" -# helper_libtestdriver1_make_drivers "$loc_accel_list" "$loc_extra_list" -# 3b. Call helper_libtestdriver1_make_main "$loc_accel_list". Any -# additional arguments will be passed to make: this can be useful if -# you don't want to build everything when iterating during development. -# Example: -# helper_libtestdriver1_make_main "$loc_accel_list" -C tests test_suite_foo -# 4. Run the tests you want. - -# Adjust the configuration - for both libtestdriver1 and main library, -# as they should have the same PSA_WANT macros. -helper_libtestdriver1_adjust_config () { - base_config=$1 - # Select the base configuration - if [ "$base_config" != "default" ]; then - scripts/config.py "$base_config" - fi - - # Enable PSA-based config (necessary to use drivers) - scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG - - # Dynamic secure element support is a deprecated feature and needs to be disabled here. - # This is done to have the same form of psa_key_attributes_s for libdriver and library. - scripts/config.py unset MBEDTLS_PSA_CRYPTO_SE_C - - # If threading is enabled on the normal build, then we need to enable it in the drivers as well, - # otherwise we will end up running multithreaded tests without mutexes to protect them. - if scripts/config.py get MBEDTLS_THREADING_C; then - scripts/config.py -f "$CONFIG_TEST_DRIVER_H" set MBEDTLS_THREADING_C - fi - - if scripts/config.py get MBEDTLS_THREADING_PTHREAD; then - scripts/config.py -f "$CONFIG_TEST_DRIVER_H" set MBEDTLS_THREADING_PTHREAD - fi -} - -# When called with no parameter this function disables all builtin curves. -# The function optionally accepts 1 parameter: a space-separated list of the -# curves that should be kept enabled. -helper_disable_builtin_curves () { - allowed_list="${1:-}" - scripts/config.py unset-all "MBEDTLS_ECP_DP_[0-9A-Z_a-z]*_ENABLED" - - for curve in $allowed_list; do - scripts/config.py set $curve - done -} - -# Helper returning the list of supported elliptic curves from CRYPTO_CONFIG_H, -# without the "PSA_WANT_" prefix. This becomes handy for accelerating curves -# in the following helpers. -helper_get_psa_curve_list () { - loc_list="" - for item in $(sed -n 's/^#define PSA_WANT_\(ECC_[0-9A-Z_a-z]*\).*/\1/p' <"$CRYPTO_CONFIG_H"); do - loc_list="$loc_list $item" - done - - echo "$loc_list" -} - -# Helper returning the list of supported DH groups from CRYPTO_CONFIG_H, -# without the "PSA_WANT_" prefix. This becomes handy for accelerating DH groups -# in the following helpers. -helper_get_psa_dh_group_list () { - loc_list="" - for item in $(sed -n 's/^#define PSA_WANT_\(DH_RFC7919_[0-9]*\).*/\1/p' <"$CRYPTO_CONFIG_H"); do - loc_list="$loc_list $item" - done - - echo "$loc_list" -} - -# Get the list of uncommented PSA_WANT_KEY_TYPE_xxx_ from CRYPTO_CONFIG_H. This -# is useful to easily get a list of key type symbols to accelerate. -# The function accepts a single argument which is the key type: ECC, DH, RSA. -helper_get_psa_key_type_list () { - key_type="$1" - loc_list="" - for item in $(sed -n "s/^#define PSA_WANT_\(KEY_TYPE_${key_type}_[0-9A-Z_a-z]*\).*/\1/p" <"$CRYPTO_CONFIG_H"); do - # Skip DERIVE for elliptic keys since there is no driver dispatch for - # it so it cannot be accelerated. - if [ "$item" != "KEY_TYPE_ECC_KEY_PAIR_DERIVE" ]; then - loc_list="$loc_list $item" - fi - done - - echo "$loc_list" -} - -# Build the drivers library libtestdriver1.a (with ASan). -# -# Parameters: -# 1. a space-separated list of things to accelerate; -# 2. optional: a space-separate list of things to also support. -# Here "things" are PSA_WANT_ symbols but with PSA_WANT_ removed. -helper_libtestdriver1_make_drivers () { - loc_accel_flags=$( echo "$1 ${2-}" | sed 's/[^ ]* */-DLIBTESTDRIVER1_MBEDTLS_PSA_ACCEL_&/g' ) - make CC=$ASAN_CC -C tests libtestdriver1.a CFLAGS=" $ASAN_CFLAGS $loc_accel_flags" LDFLAGS="$ASAN_CFLAGS" -} - -# Build the main libraries, programs and tests, -# linking to the drivers library (with ASan). -# -# Parameters: -# 1. a space-separated list of things to accelerate; -# *. remaining arguments if any are passed directly to make -# (examples: lib, -C tests test_suite_xxx, etc.) -# Here "things" are PSA_WANT_ symbols but with PSA_WANT_ removed. -helper_libtestdriver1_make_main () { - loc_accel_list=$1 - shift - - # we need flags both with and without the LIBTESTDRIVER1_ prefix - loc_accel_flags=$( echo "$loc_accel_list" | sed 's/[^ ]* */-DLIBTESTDRIVER1_MBEDTLS_PSA_ACCEL_&/g' ) - loc_accel_flags="$loc_accel_flags $( echo "$loc_accel_list" | sed 's/[^ ]* */-DMBEDTLS_PSA_ACCEL_&/g' )" - make CC=$ASAN_CC CFLAGS="$ASAN_CFLAGS -I../tests/include -I../tests -I../../tests -DPSA_CRYPTO_DRIVER_TEST -DMBEDTLS_TEST_LIBTESTDRIVER1 $loc_accel_flags" LDFLAGS="-ltestdriver1 $ASAN_CFLAGS" "$@" -} - -# Include the components from components.sh +# The path is going to change when this is moved to the framework test_script_dir="${0%/*}" -for file in "$test_script_dir"/components*.sh; do - source $file -done +source "$test_script_dir"/all-core.sh -################################################################ -#### Termination -################################################################ - -post_report () { - msg "Done, cleaning up" - final_cleanup - - final_report -} - -################################################################ -#### Run all the things -################################################################ - -# Function invoked by --error-test to test error reporting. -pseudo_component_error_test () { - msg "Testing error reporting $error_test_i" - if [ $KEEP_GOING -ne 0 ]; then - echo "Expect three failing commands." - fi - # If the component doesn't run in a subshell, changing error_test_i to an - # invalid integer will cause an error in the loop that runs this function. - error_test_i=this_should_not_be_used_since_the_component_runs_in_a_subshell - # Expected error: 'grep non_existent /dev/null -> 1' - grep non_existent /dev/null - # Expected error: '! grep -q . tests/scripts/all.sh -> 1' - not grep -q . "$0" - # Expected error: 'make unknown_target -> 2' - make unknown_target - false "this should not be executed" -} - -# Run one component and clean up afterwards. -run_component () { - current_component="$1" - export MBEDTLS_TEST_CONFIGURATION="$current_component" - - # Unconditionally create a seedfile that's sufficiently long. - # Do this before each component, because a previous component may - # have messed it up or shortened it. - local dd_cmd - dd_cmd=(dd if=/dev/urandom of=./tests/seedfile bs=64 count=1) - case $OSTYPE in - linux*|freebsd*|openbsd*) dd_cmd+=(status=none) - esac - "${dd_cmd[@]}" - - # Run the component in a subshell, with error trapping and output - # redirection set up based on the relevant options. - if [ $KEEP_GOING -eq 1 ]; then - # We want to keep running if the subshell fails, so 'set -e' must - # be off when the subshell runs. - set +e - fi - ( - if [ $QUIET -eq 1 ]; then - # msg() will be silenced, so just print the component name here. - echo "${current_component#component_}" - exec >/dev/null - fi - if [ $KEEP_GOING -eq 1 ]; then - # Keep "set -e" off, and run an ERR trap instead to record failures. - set -E - trap err_trap ERR - fi - # The next line is what runs the component - "$@" - if [ $KEEP_GOING -eq 1 ]; then - trap - ERR - exit $last_failure_status - fi - ) - component_status=$? - if [ $KEEP_GOING -eq 1 ]; then - set -e - if [ $component_status -ne 0 ]; then - failure_count=$((failure_count + 1)) - fi - fi - - # Restore the build tree to a clean state. - cleanup - unset current_component -} - -# Preliminary setup -pre_check_environment -pre_initialize_variables -pre_parse_command_line "$@" - -setup_quiet_wrappers -pre_check_git -pre_restore_files -pre_back_up - -build_status=0 -if [ $KEEP_GOING -eq 1 ]; then - pre_setup_keep_going -fi -pre_prepare_outcome_file -pre_print_configuration -pre_check_tools -cleanup -if in_mbedtls_repo; then - pre_generate_files -fi - -# Run the requested tests. -for ((error_test_i=1; error_test_i <= error_test; error_test_i++)); do - run_component pseudo_component_error_test -done -unset error_test_i -for component in $RUN_COMPONENTS; do - run_component "component_$component" -done - -# We're done. -post_report +main "$@" diff --git a/tests/scripts/components-compiler.sh b/tests/scripts/components-compiler.sh index 5badabbc56..d89bbedae4 100644 --- a/tests/scripts/components-compiler.sh +++ b/tests/scripts/components-compiler.sh @@ -18,7 +18,7 @@ component_build_tfm_armcc () { cp configs/config-tfm.h "$CONFIG_H" msg "build: TF-M config, armclang armv7-m thumb2" - armc6_build_test "--target=arm-arm-none-eabi -march=armv7-m -mthumb -Os -std=c99 -Werror -Wall -Wextra -Wwrite-strings -Wpointer-arith -Wimplicit-fallthrough -Wshadow -Wvla -Wformat=2 -Wno-format-nonliteral -Wshadow -Wasm-operand-widths -Wunused -I../tests/include/spe" + helper_armc6_build_test "--target=arm-arm-none-eabi -march=armv7-m -mthumb -Os -std=c99 -Werror -Wall -Wextra -Wwrite-strings -Wpointer-arith -Wimplicit-fallthrough -Wshadow -Wvla -Wformat=2 -Wno-format-nonliteral -Wshadow -Wasm-operand-widths -Wunused -I../tests/include/spe" } test_build_opt () { diff --git a/tests/scripts/components-configuration-crypto.sh b/tests/scripts/components-configuration-crypto.sh index d2f7c22db5..db83e765f5 100644 --- a/tests/scripts/components-configuration-crypto.sh +++ b/tests/scripts/components-configuration-crypto.sh @@ -2849,12 +2849,12 @@ component_build_aes_aesce_armcc () { msg "AESCE, build with default configuration." scripts/config.py set MBEDTLS_AESCE_C scripts/config.py unset MBEDTLS_AES_USE_HARDWARE_ONLY - armc6_build_test "-O1 --target=aarch64-arm-none-eabi -march=armv8-a+crypto" + helper_armc6_build_test "-O1 --target=aarch64-arm-none-eabi -march=armv8-a+crypto" msg "AESCE, build AESCE only" scripts/config.py set MBEDTLS_AESCE_C scripts/config.py set MBEDTLS_AES_USE_HARDWARE_ONLY - armc6_build_test "-O1 --target=aarch64-arm-none-eabi -march=armv8-a+crypto" + helper_armc6_build_test "-O1 --target=aarch64-arm-none-eabi -march=armv8-a+crypto" } support_build_aes_aesce_armcc () { @@ -3084,7 +3084,7 @@ component_test_block_cipher_no_decrypt_aesce_armcc () { # test AESCE baremetal build scripts/config.py set MBEDTLS_AESCE_C msg "build: default config + BLOCK_CIPHER_NO_DECRYPT with AESCE" - armc6_build_test "-O1 --target=aarch64-arm-none-eabi -march=armv8-a+crypto -Werror -Wall -Wextra" + helper_armc6_build_test "-O1 --target=aarch64-arm-none-eabi -march=armv8-a+crypto -Werror -Wall -Wextra" # Make sure we don't have mbedtls_xxx_setkey_dec in AES/ARIA/CAMELLIA not grep mbedtls_aes_setkey_dec library/aes.o diff --git a/tests/scripts/components-platform.sh b/tests/scripts/components-platform.sh index b104428278..94d5fa1b1c 100644 --- a/tests/scripts/components-platform.sh +++ b/tests/scripts/components-platform.sh @@ -461,32 +461,32 @@ component_build_armcc () { # Compile mostly with -O1 since some Arm inline assembly is disabled for -O0. # ARM Compiler 6 - Target ARMv7-A - armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv7-a" + helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv7-a" # ARM Compiler 6 - Target ARMv7-M - armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv7-m" + helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv7-m" # ARM Compiler 6 - Target ARMv7-M+DSP - armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv7-m+dsp" + helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv7-m+dsp" # ARM Compiler 6 - Target ARMv8-A - AArch32 - armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv8.2-a" + helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv8.2-a" # ARM Compiler 6 - Target ARMv8-M - armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv8-m.main" + helper_armc6_build_test "-O1 --target=arm-arm-none-eabi -march=armv8-m.main" # ARM Compiler 6 - Target Cortex-M0 - no optimisation - armc6_build_test "-O0 --target=arm-arm-none-eabi -mcpu=cortex-m0" + helper_armc6_build_test "-O0 --target=arm-arm-none-eabi -mcpu=cortex-m0" # ARM Compiler 6 - Target Cortex-M0 - armc6_build_test "-Os --target=arm-arm-none-eabi -mcpu=cortex-m0" + helper_armc6_build_test "-Os --target=arm-arm-none-eabi -mcpu=cortex-m0" # ARM Compiler 6 - Target ARMv8.2-A - AArch64 # # Re-enable MBEDTLS_AESCE_C as this should be supported by the version of armclang # that we have in our CI scripts/config.py set MBEDTLS_AESCE_C - armc6_build_test "-O1 --target=aarch64-arm-none-eabi -march=armv8.2-a+crypto" + helper_armc6_build_test "-O1 --target=aarch64-arm-none-eabi -march=armv8.2-a+crypto" } support_build_armcc () {