1
0
mirror of https://github.com/Kitware/CMake.git synced 2025-10-21 14:40:48 +08:00

ctest: add option for output truncation

Add `--test-output-truncation` to `ctest`. This option can be used to
customize which part of the test output is being truncated. Currently
supported values are `tail`, `middle` and `head`.

Also add equivalent `CTEST_CUSTOM_TEST_OUTPUT_TRUNCATION` variable.

Fixes: #23206
This commit is contained in:
Frank Winklmeier
2022-03-07 09:28:55 +01:00
committed by Brad King
parent 359e5b17d8
commit 140704d443
26 changed files with 260 additions and 22 deletions

View File

@@ -1573,6 +1573,7 @@ syn keyword cmakeVariable contained
\ CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS
\ CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS
\ CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE
\ CTEST_CUSTOM_TEST_OUTPUT_TRUNCATION
\ CTEST_CUSTOM_MEMCHECK_IGNORE
\ CTEST_CUSTOM_POST_MEMCHECK
\ CTEST_CUSTOM_POST_TEST

View File

@@ -172,8 +172,9 @@ The options are:
affected. Summary info detailing the percentage of passing tests is also
unaffected by the ``QUIET`` option.
See also the :variable:`CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE`
and :variable:`CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE` variables.
See also the :variable:`CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE`,
:variable:`CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE` and
:variable:`CTEST_CUSTOM_TEST_OUTPUT_TRUNCATION` variables.
.. _`Additional Test Measurements`:

View File

@@ -694,6 +694,12 @@ that may contain the following fields:
bytes. Equivalent to passing ``--test-output-size-failed`` on the
command line.
``testOutputTruncation``
An optional string specifying the test output truncation mode. Equivalent
to passing ``--test-output-truncation`` on the command line."
This is allowed in preset files specifying version ``5`` or above.
``maxTestNameWidth``
An optional integer specifying the maximum width of a test name to

View File

@@ -636,6 +636,7 @@ Variables for CTest
/variable/CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS
/variable/CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS
/variable/CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE
/variable/CTEST_CUSTOM_TEST_OUTPUT_TRUNCATION
/variable/CTEST_CUSTOM_MEMCHECK_IGNORE
/variable/CTEST_CUSTOM_POST_MEMCHECK
/variable/CTEST_CUSTOM_POST_TEST

View File

@@ -362,6 +362,10 @@ Specify the directory in which to look for tests.
``--test-output-size-failed <size>``
Limit the output for failed tests to ``<size>`` bytes.
``--test-output-truncation <mode>``
Truncate 'tail' (default), 'middle' or 'head' of test output once maximum
output size is reached.
``--overwrite``
Overwrite CTest configuration option.

View File

@@ -688,6 +688,28 @@
"additionalProperties": false
}
},
"testPresetsItemsV5": {
"type": "array",
"description": "An optional array of test preset objects. Used to specify arguments to ctest. Available in version 5 and higher.",
"items": {
"type": "object",
"properties": {
"output": {
"type": "object",
"description": "An optional object specifying output options.",
"properties": {
"testOutputTruncation": {
"type": "string",
"description": "An optional string specifying the test output truncation mode. Equivalent to passing --test-output-truncation on the command line. Must be one of the following values: \"tail\", \"middle\", or \"head\".",
"enum": [
"tail", "middle", "head"
]
}
}
}
}
}
},
"testPresetsItemsV3": {
"type": "array",
"description": "An optional array of test preset objects. Used to specify arguments to ctest. Available in version 3 and higher.",
@@ -1017,7 +1039,8 @@
"description": "An optional array of test preset objects. Used to specify arguments to ctest. Available in version 5 and higher.",
"allOf": [
{ "$ref": "#/definitions/testPresetsItemsV2" },
{ "$ref": "#/definitions/testPresetsItemsV3" }
{ "$ref": "#/definitions/testPresetsItemsV3" },
{ "$ref": "#/definitions/testPresetsItemsV5" }
],
"items": {
"type": "object",

View File

@@ -3,7 +3,8 @@ CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE
When saving a failing test's output, this is the maximum size, in bytes, that
will be collected by the :command:`ctest_test` command. Defaults to 307200
(300 KiB).
(300 KiB). See :variable:`CTEST_CUSTOM_TEST_OUTPUT_TRUNCATION` for possible
truncation modes.
If a test's output contains the literal string "CTEST_FULL_OUTPUT",
the output will not be truncated and may exceed the maximum size.

View File

@@ -3,7 +3,8 @@ CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE
When saving a passing test's output, this is the maximum size, in bytes, that
will be collected by the :command:`ctest_test` command. Defaults to 1024
(1 KiB).
(1 KiB). See :variable:`CTEST_CUSTOM_TEST_OUTPUT_TRUNCATION` for possible
truncation modes.
If a test's output contains the literal string "CTEST_FULL_OUTPUT",
the output will not be truncated and may exceed the maximum size.

View File

@@ -0,0 +1,12 @@
CTEST_CUSTOM_TEST_OUTPUT_TRUNCATION
-----------------------------------
.. versionadded:: 3.24
Set the test output truncation mode in case a maximum size is configured
via the :variable:`CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE` or
:variable:`CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE` variables.
By default the ``tail`` of the output will be truncated. Other possible
values are ``middle`` and ``head``.
.. include:: CTEST_CUSTOM_XXX.txt

View File

@@ -371,7 +371,8 @@ void cmCTestMemCheckHandler::GenerateCTestXML(cmXMLWriter& xml)
}
this->CleanTestOutput(
memcheckstr,
static_cast<size_t>(this->CustomMaximumFailedTestOutputSize));
static_cast<size_t>(this->CustomMaximumFailedTestOutputSize),
this->TestOutputTruncation);
this->WriteTestResultHeader(xml, result);
xml.StartElement("Results");
int memoryErrors = 0;

View File

@@ -277,7 +277,8 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
static_cast<size_t>(
this->TestResult.Status == cmCTestTestHandler::COMPLETED
? this->TestHandler->CustomMaximumPassedTestOutputSize
: this->TestHandler->CustomMaximumFailedTestOutputSize));
: this->TestHandler->CustomMaximumFailedTestOutputSize),
this->TestHandler->TestOutputTruncation);
}
this->TestResult.Reason = reason;
if (this->TestHandler->LogFile) {

View File

@@ -281,6 +281,7 @@ cmCTestTestHandler::cmCTestTestHandler()
this->CustomMaximumPassedTestOutputSize = 1 * 1024;
this->CustomMaximumFailedTestOutputSize = 300 * 1024;
this->TestOutputTruncation = cmCTestTypes::TruncationMode::Tail;
this->MemCheck = false;
@@ -325,6 +326,7 @@ void cmCTestTestHandler::Initialize()
this->CustomPostTest.clear();
this->CustomMaximumPassedTestOutputSize = 1 * 1024;
this->CustomMaximumFailedTestOutputSize = 300 * 1024;
this->TestOutputTruncation = cmCTestTypes::TruncationMode::Tail;
this->TestsToRun.clear();
@@ -358,6 +360,11 @@ void cmCTestTestHandler::PopulateCustomVectors(cmMakefile* mf)
this->CTest->PopulateCustomInteger(
mf, "CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE",
this->CustomMaximumFailedTestOutputSize);
cmValue dval = mf->GetDefinition("CTEST_CUSTOM_TEST_OUTPUT_TRUNCATION");
if (dval) {
this->SetTestOutputTruncation(dval);
}
}
int cmCTestTestHandler::PreProcessHandler()
@@ -2076,6 +2083,20 @@ void cmCTestTestHandler::SetExcludeRegExp(const std::string& arg)
this->ExcludeRegExp = arg;
}
bool cmCTestTestHandler::SetTestOutputTruncation(const std::string& mode)
{
if (mode == "tail") {
this->TestOutputTruncation = cmCTestTypes::TruncationMode::Tail;
} else if (mode == "middle") {
this->TestOutputTruncation = cmCTestTypes::TruncationMode::Middle;
} else if (mode == "head") {
this->TestOutputTruncation = cmCTestTypes::TruncationMode::Head;
} else {
return false;
}
return true;
}
void cmCTestTestHandler::SetTestsToRunInformation(cmValue in)
{
if (!in) {
@@ -2094,7 +2115,8 @@ void cmCTestTestHandler::SetTestsToRunInformation(cmValue in)
}
}
void cmCTestTestHandler::CleanTestOutput(std::string& output, size_t length)
void cmCTestTestHandler::CleanTestOutput(std::string& output, size_t length,
cmCTestTypes::TruncationMode truncate)
{
if (!length || length >= output.size() ||
output.find("CTEST_FULL_OUTPUT") != std::string::npos) {
@@ -2122,20 +2144,29 @@ void cmCTestTestHandler::CleanTestOutput(std::string& output, size_t length)
return current;
};
// Truncate at given length respecting UTF-8 words
// Truncation message.
const std::string msg =
"\n[This part of the test output was removed since it "
"exceeds the threshold of " +
std::to_string(length) + " bytes.]\n";
char const* const begin = output.c_str();
char const* const end = begin + output.size();
char const* current = utf8_advance(begin, end, length);
output.erase(current - begin);
// Append truncation message.
std::ostringstream msg;
msg << "...\n"
"The rest of the test output was removed since it exceeds the "
"threshold "
"of "
<< length << " bytes.\n";
output += msg.str();
// Erase head, middle or tail of output.
if (truncate == cmCTestTypes::TruncationMode::Head) {
char const* current = utf8_advance(begin, end, output.size() - length);
output.erase(0, current - begin);
output.insert(0, msg + "...");
} else if (truncate == cmCTestTypes::TruncationMode::Middle) {
char const* current = utf8_advance(begin, end, length / 2);
output.erase(current - begin, output.size() - length);
output.insert(current - begin, "..." + msg + "...");
} else { // default or "tail"
char const* current = utf8_advance(begin, end, length);
output.erase(current - begin);
output += ("..." + msg);
}
}
bool cmCTestTestHandler::SetTestsProperties(

View File

@@ -19,6 +19,7 @@
#include "cmCTest.h"
#include "cmCTestGenericHandler.h"
#include "cmCTestResourceSpec.h"
#include "cmCTestTypes.h"
#include "cmDuration.h"
#include "cmListFileCache.h"
#include "cmValue.h"
@@ -32,6 +33,7 @@ class cmXMLWriter;
*/
class cmCTestTestHandler : public cmCTestGenericHandler
{
friend class cmCTest;
friend class cmCTestRunTest;
friend class cmCTestMultiProcessHandler;
@@ -80,6 +82,9 @@ public:
this->CustomMaximumFailedTestOutputSize = n;
}
//! Set test output truncation mode. Return false if unknown mode.
bool SetTestOutputTruncation(const std::string& mode);
//! pass the -I argument down
void SetTestsToRunInformation(cmValue);
@@ -242,8 +247,9 @@ protected:
void AttachFile(cmXMLWriter& xml, std::string const& file,
std::string const& name);
//! Clean test output to specified length
void CleanTestOutput(std::string& output, size_t length);
//! Clean test output to specified length and truncation mode
void CleanTestOutput(std::string& output, size_t length,
cmCTestTypes::TruncationMode truncate);
cmDuration ElapsedTestingTime;
@@ -258,6 +264,7 @@ protected:
bool MemCheck;
int CustomMaximumPassedTestOutputSize;
int CustomMaximumFailedTestOutputSize;
cmCTestTypes::TruncationMode TestOutputTruncation;
int MaxIndex;
public:

View File

@@ -0,0 +1,16 @@
/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying
file Copyright.txt or https://cmake.org/licensing for details. */
#pragma once
#include "cmConfigure.h" // IWYU pragma: keep
namespace cmCTestTypes {
enum class TruncationMode
{ // Test output truncation mode
Tail,
Middle,
Head
};
}

View File

@@ -773,6 +773,8 @@ cmCMakePresetsGraph::TestPreset::VisitPresetInherit(
parentOutput.MaxPassedTestOutputSize);
InheritOptionalValue(output.MaxFailedTestOutputSize,
parentOutput.MaxFailedTestOutputSize);
InheritOptionalValue(output.TestOutputTruncation,
parentOutput.TestOutputTruncation);
InheritOptionalValue(output.MaxTestNameWidth,
parentOutput.MaxTestNameWidth);
} else {
@@ -1027,6 +1029,9 @@ const char* cmCMakePresetsGraph::ResultToString(ReadFileResult result)
"support.";
case ReadFileResult::CYCLIC_INCLUDE:
return "Cyclic include among preset files";
case ReadFileResult::TEST_OUTPUT_TRUNCATION_UNSUPPORTED:
return "File version must be 5 or higher for testOutputTruncation "
"preset support.";
}
return "Unknown error";

View File

@@ -14,6 +14,8 @@
#include <cm/optional>
#include "CTest/cmCTestTypes.h"
enum class PackageResolveMode;
class cmCMakePresetsGraph
@@ -47,6 +49,7 @@ public:
CONDITION_UNSUPPORTED,
TOOLCHAIN_FILE_UNSUPPORTED,
CYCLIC_INCLUDE,
TEST_OUTPUT_TRUNCATION_UNSUPPORTED,
};
enum class ArchToolsetStrategy
@@ -226,6 +229,7 @@ public:
cm::optional<bool> SubprojectSummary;
cm::optional<int> MaxPassedTestOutputSize;
cm::optional<int> MaxFailedTestOutputSize;
cm::optional<cmCTestTypes::TruncationMode> TestOutputTruncation;
cm::optional<int> MaxTestNameWidth;
};

View File

@@ -568,6 +568,11 @@ cmCMakePresetsGraph::ReadFileResult cmCMakePresetsGraph::ReadJSONFile(
return ReadFileResult::CONDITION_UNSUPPORTED;
}
// Support for TestOutputTruncation added in version 5.
if (v < 5 && preset.Output) {
return ReadFileResult::TEST_OUTPUT_TRUNCATION_UNSUPPORTED;
}
this->TestPresetOrder.push_back(preset.Name);
}

View File

@@ -16,6 +16,8 @@
#include "cmCMakePresetsGraphInternal.h"
#include "cmJSONHelpers.h"
#include "CTest/cmCTestTypes.h"
namespace {
using ReadFileResult = cmCMakePresetsGraph::ReadFileResult;
using TestPreset = cmCMakePresetsGraph::TestPreset;
@@ -55,6 +57,40 @@ auto const TestPresetOptionalOutputVerbosityHelper =
ReadFileResult>(ReadFileResult::READ_OK,
TestPresetOutputVerbosityHelper);
ReadFileResult TestPresetOutputTruncationHelper(
cmCTestTypes::TruncationMode& out, const Json::Value* value)
{
if (!value) {
out = cmCTestTypes::TruncationMode::Tail;
return ReadFileResult::READ_OK;
}
if (!value->isString()) {
return ReadFileResult::INVALID_PRESET;
}
if (value->asString() == "tail") {
out = cmCTestTypes::TruncationMode::Tail;
return ReadFileResult::READ_OK;
}
if (value->asString() == "middle") {
out = cmCTestTypes::TruncationMode::Middle;
return ReadFileResult::READ_OK;
}
if (value->asString() == "head") {
out = cmCTestTypes::TruncationMode::Head;
return ReadFileResult::READ_OK;
}
return ReadFileResult::INVALID_PRESET;
}
auto const TestPresetOptionalTruncationHelper =
cmJSONOptionalHelper<cmCTestTypes::TruncationMode, ReadFileResult>(
ReadFileResult::READ_OK, TestPresetOutputTruncationHelper);
auto const TestPresetOptionalOutputHelper =
cmJSONOptionalHelper<TestPreset::OutputOptions, ReadFileResult>(
ReadFileResult::READ_OK,
@@ -83,6 +119,9 @@ auto const TestPresetOptionalOutputHelper =
.Bind("maxFailedTestOutputSize"_s,
&TestPreset::OutputOptions::MaxFailedTestOutputSize,
cmCMakePresetsGraphInternal::PresetOptionalIntHelper, false)
.Bind("testOutputTruncation"_s,
&TestPreset::OutputOptions::TestOutputTruncation,
TestPresetOptionalTruncationHelper, false)
.Bind("maxTestNameWidth"_s, &TestPreset::OutputOptions::MaxTestNameWidth,
cmCMakePresetsGraphInternal::PresetOptionalIntHelper, false));

View File

@@ -2036,6 +2036,13 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
"Invalid value for '--test-output-size-failed': " << args[i]
<< "\n");
}
} else if (this->CheckArgument(arg, "--test-output-truncation"_s) &&
i < args.size() - 1) {
i++;
if (!this->Impl->TestHandler.SetTestOutputTruncation(args[i])) {
errormsg = "Invalid value for '--test-output-truncation': " + args[i];
return false;
}
} else if (this->CheckArgument(arg, "-N"_s, "--show-only")) {
this->Impl->ShowOnly = true;
} else if (cmHasLiteralPrefix(arg, "--show-only=")) {
@@ -2464,6 +2471,11 @@ bool cmCTest::SetArgsFromPreset(const std::string& presetName,
*expandedPreset->Output->MaxFailedTestOutputSize);
}
if (expandedPreset->Output->TestOutputTruncation) {
this->Impl->TestHandler.TestOutputTruncation =
*expandedPreset->Output->TestOutputTruncation;
}
if (expandedPreset->Output->MaxTestNameWidth) {
this->Impl->MaxTestNameWidth = *expandedPreset->Output->MaxTestNameWidth;
}

View File

@@ -44,6 +44,9 @@ static const char* cmDocumentationOptions[][2] = {
{ "--test-output-size-failed <size>",
"Limit the output for failed tests "
"to <size> bytes" },
{ "--test-output-truncation <mode>",
"Truncate 'tail' (default), 'middle' or 'head' of test output once "
"maximum output size is reached" },
{ "-F", "Enable failover." },
{ "-j <jobs>, --parallel <jobs>",
"Run the tests in parallel using the "

View File

@@ -48,7 +48,8 @@
"quiet": false,
"outputLogFile": "",
"labelSummary": true,
"subprojectSummary": true
"subprojectSummary": true,
"testOutputTruncation": "tail"
},
"filter": {
"include": {

View File

@@ -274,6 +274,27 @@ function(run_TestOutputSize)
endfunction()
run_TestOutputSize()
# Test --test-output-truncation
function(run_TestOutputTruncation mode expected)
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/TestOutputTruncation_${mode})
set(RunCMake_TEST_NO_CLEAN 1)
set(TRUNCATED_OUTPUT ${expected}) # used in TestOutputTruncation-check.cmake
file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "
add_test(Truncation_${mode} \"${CMAKE_COMMAND}\" -E echo 123456789)
")
run_cmake_command(TestOutputTruncation
${CMAKE_CTEST_COMMAND} -M Experimental -T Test
--no-compress-output
--test-output-size-passed 5
--test-output-truncation ${mode}
)
endfunction()
run_TestOutputTruncation("head" "\\.\\.\\.6789")
run_TestOutputTruncation("middle" "12\\.\\.\\..*\\.\\.\\.89")
run_TestOutputTruncation("tail" "12345\.\.\.")
# Test --stop-on-failure
function(run_stop_on_failure)
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/stop-on-failure)

View File

@@ -0,0 +1,12 @@
file(GLOB test_xml_file "${RunCMake_TEST_BINARY_DIR}/Testing/*/Test.xml")
if(test_xml_file)
file(READ "${test_xml_file}" test_xml LIMIT 4096)
if("${test_xml}" MATCHES [[(<Test Status="passed">.*</Test>)]])
set(test_result "${CMAKE_MATCH_1}")
endif()
if(NOT "${test_result}" MATCHES "<Value>.*${TRUNCATED_OUTPUT}.*</Value>")
set(RunCMake_TEST_FAILED "Test output truncation failed:\n ${test_result}\nExpected: ${TRUNCATED_OUTPUT}")
endif()
else()
set(RunCMake_TEST_FAILED "Test.xml not found")
endif()

View File

@@ -0,0 +1 @@
^Cannot find file: .*/Tests/RunCMake/CTestCommandLine/TestOutputTruncation.*/DartConfiguration.tcl

View File

@@ -80,6 +80,23 @@ add_test(NAME FailingTest COMMAND ${CMAKE_COMMAND} -E no_such_command)
endfunction()
run_TestOutputSize()
# Test --test-output-truncation
function(run_TestOutputTruncation mode expected)
set(CASE_CTEST_TEST_ARGS EXCLUDE RunCMakeVersion)
set(TRUNCATED_OUTPUT ${expected}) # used in TestOutputTruncation-check.cmake
set(CASE_TEST_PREFIX_CODE [[
set( CTEST_CUSTOM_TEST_OUTPUT_TRUNCATION${mode})
]])
set(CASE_CMAKELISTS_SUFFIX_CODE [[
add_test(NAME Truncation_${mode} COMMAND ${CMAKE_COMMAND} -E echo 123456789)
]])
run_ctest(TestOutputTruncation)
endfunction()
run_TestOutputTruncation("head" "...6789")
run_TestOutputTruncation("middle" "12....*...89")
run_TestOutputTruncation("tail" "12345...")
run_ctest_test(TestRepeatBad1 REPEAT UNKNOWN:3)
run_ctest_test(TestRepeatBad2 REPEAT UNTIL_FAIL:-1)

View File

@@ -0,0 +1,12 @@
file(GLOB test_xml_file "${RunCMake_TEST_BINARY_DIR}/Testing/*/Test.xml")
if(test_xml_file)
file(READ "${test_xml_file}" test_xml LIMIT 4096)
if("${test_xml}" MATCHES [[(<Test Status="passed">.*</Test>)]])
set(test_result "${CMAKE_MATCH_1}")
endif()
if(NOT "${test_result}" MATCHES "<Value>.*${TRUNCATED_OUTPUT}.*</Value>")
set(RunCMake_TEST_FAILED "Test output truncation failed:\n ${test_result}\nExpected: ${TRUNCATED_OUTPUT}")
endif()
else()
set(RunCMake_TEST_FAILED "Test.xml not found")
endif()