Kitware/CMake
1
0
Fork 0
mirror of https://github.com/Kitware/CMake.git synced 2025-10-14 02:08:27 +08:00
Code Issues Projects Releases Wiki Activity

ctest: add support for attaching files to tests at run time

Browse Source 
Allow tests to specify files to upload at runtime. Previously this was
only possible to specify at configure time with the ATTACHED_FILES
test properties.

This commit also fixes a bug in how our test data tarballs were generated
by CTest. Previously, if you tried to attach a file outside of the binary
directory, CTest would generate a tar file with a relative path, and tar
would not allow you to extract it. We resolve this problem by creating
tar files with a flat directory structure instead.

Fixes: #22284
This commit is contained in:
Zack Galbreath
2021-06-04 11:22:34 -04:00
parent acb25d50d9
commit cbcb92d1cb
9 changed files with 140 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
Help/command/ctest_test.rst
View File

@@ -248,5 +248,14 @@ separate from the interactive comparison UI.
Attached Files
""""""""""""""
To associate other types of files with a test, use the
:prop_test:`ATTACHED_FILES` or :prop_test:`ATTACHED_FILES_ON_FAIL` test properties.
The following example demonstrates how to upload non-image files to CDash.
.. code-block:: c++
std::cout <<
"<DartMeasurementFile type=\"file\" name=\"MyTestInputData\">" <<
"/dir/to/data.csv</DartMeasurementFile>" << std::endl;
If the name of the file to upload is known at configure time, you can use the
:prop_test:`ATTACHED_FILES` or :prop_test:`ATTACHED_FILES_ON_FAIL` test
properties instead.

5
Help/release/dev/ctest-measurements-docs.rst Normal file
View File

@@ -0,0 +1,5 @@
ctest-measurements-docs
-----------------------
* :manual:`ctest(1)` gained documentation for its ability to capture
:ref:`Additional Test Measurements`.

8
Help/release/dev/ctest-runtime-files.rst Normal file
View File

@@ -0,0 +1,8 @@
ctest-runtime-files
-------------------
* :manual:`ctest(1)` learned to recognize files attached to a test at run time.
Previously it was only possible to attach files to tests at configure time
by using the :prop_test:`ATTACHED_FILES` or
:prop_test:`ATTACHED_FILES_ON_FAIL` test properties.
See :ref:`Additional Test Measurements` for more information.

112
Source/CTest/cmCTestTestHandler.cxx
View File

@@ -1550,19 +1550,29 @@ void cmCTestTestHandler::AttachFiles(cmXMLWriter& xml,
result.Properties->AttachOnFail.end());
}
for (std::string const& file : result.Properties->AttachedFiles) {
const std::string& base64 = this->CTest->Base64GzipEncodeFile(file);
std::string const fname = cmSystemTools::GetFilenameName(file);
xml.StartElement("NamedMeasurement");
xml.Attribute("name", "Attached File");
xml.Attribute("encoding", "base64");
xml.Attribute("compression", "tar/gzip");
xml.Attribute("filename", fname);
xml.Attribute("type", "file");
xml.Element("Value", base64);
xml.EndElement(); // NamedMeasurement
this->AttachFile(xml, file, "");
}
}
void cmCTestTestHandler::AttachFile(cmXMLWriter& xml, std::string const& file,
std::string const& name)
{
const std::string& base64 = this->CTest->Base64GzipEncodeFile(file);
std::string const fname = cmSystemTools::GetFilenameName(file);
xml.StartElement("NamedMeasurement");
std::string measurement_name = name;
if (measurement_name.empty()) {
measurement_name = "Attached File";
}
xml.Attribute("name", measurement_name);
xml.Attribute("encoding", "base64");
xml.Attribute("compression", "tar/gzip");
xml.Attribute("filename", fname);
xml.Attribute("type", "file");
xml.Element("Value", base64);
xml.EndElement(); // NamedMeasurement
}
int cmCTestTestHandler::ExecuteCommands(std::vector<std::string>& vec)
{
for (std::string const& it : vec) {
@@ -2041,11 +2051,11 @@ void cmCTestTestHandler::GenerateRegressionImages(cmXMLWriter& xml,
cmCTest::CleanString(measurementfile.match(5));
if (cmSystemTools::FileExists(filename)) {
long len = cmSystemTools::FileLength(filename);
std::string k1 = measurementfile.match(1);
std::string v1 = measurementfile.match(2);
std::string k2 = measurementfile.match(3);
std::string v2 = measurementfile.match(4);
if (len == 0) {
std::string k1 = measurementfile.match(1);
std::string v1 = measurementfile.match(2);
std::string k2 = measurementfile.match(3);
std::string v2 = measurementfile.match(4);
if (cmSystemTools::LowerCase(k1) == "type") {
v1 = "text/string";
}
@@ -2060,35 +2070,53 @@ void cmCTestTestHandler::GenerateRegressionImages(cmXMLWriter& xml,
xml.Element("Value", "Image " + filename + " is empty");
xml.EndElement();
} else {
cmsys::ifstream ifs(filename.c_str(),
std::ios::in
#ifdef _WIN32
| std::ios::binary
#endif
);
auto file_buffer = cm::make_unique<unsigned char[]>(len + 1);
ifs.read(reinterpret_cast<char*>(file_buffer.get()), len);
auto encoded_buffer = cm::make_unique<unsigned char[]>(
static_cast<int>(static_cast<double>(len) * 1.5 + 5.0));
size_t rlen = cmsysBase64_Encode(file_buffer.get(), len,
encoded_buffer.get(), 1);
xml.StartElement("NamedMeasurement");
xml.Attribute(measurementfile.match(1).c_str(),
measurementfile.match(2));
xml.Attribute(measurementfile.match(3).c_str(),
measurementfile.match(4));
xml.Attribute("encoding", "base64");
std::ostringstream ostr;
for (size_t cc = 0; cc < rlen; cc++) {
ostr << encoded_buffer[cc];
if (cc % 60 == 0 && cc) {
ostr << std::endl;
}
std::string type;
std::string name;
if (cmSystemTools::LowerCase(k1) == "type") {
type = v1;
} else if (cmSystemTools::LowerCase(k2) == "type") {
type = v2;
}
if (cmSystemTools::LowerCase(k1) == "name") {
name = v1;
} else if (cmSystemTools::LowerCase(k2) == "name") {
name = v2;
}
if (type == "file") {
// Treat this measurement like an "ATTACHED_FILE" when the type
// is explicitly "file" (not an image).
this->AttachFile(xml, filename, name);
} else {
cmsys::ifstream ifs(filename.c_str(),
std::ios::in
#ifdef _WIN32
| std::ios::binary
#endif
);
auto file_buffer = cm::make_unique<unsigned char[]>(len + 1);
ifs.read(reinterpret_cast<char*>(file_buffer.get()), len);
auto encoded_buffer = cm::make_unique<unsigned char[]>(
static_cast<int>(static_cast<double>(len) * 1.5 + 5.0));
size_t rlen = cmsysBase64_Encode(file_buffer.get(), len,
encoded_buffer.get(), 1);
xml.StartElement("NamedMeasurement");
xml.Attribute(measurementfile.match(1).c_str(),
measurementfile.match(2));
xml.Attribute(measurementfile.match(3).c_str(),
measurementfile.match(4));
xml.Attribute("encoding", "base64");
std::ostringstream ostr;
for (size_t cc = 0; cc < rlen; cc++) {
ostr << encoded_buffer[cc];
if (cc % 60 == 0 && cc) {
ostr << std::endl;
}
}
xml.Element("Value", ostr.str());
xml.EndElement(); // NamedMeasurement
}
xml.Element("Value", ostr.str());
xml.EndElement(); // NamedMeasurement
}
} else {
int idx = 4;

2
Source/CTest/cmCTestTestHandler.h
View File

@@ -237,6 +237,8 @@ protected:
cmCTestTestResult const& result);
// Write attached test files into the xml
void AttachFiles(cmXMLWriter& xml, cmCTestTestResult& result);
void AttachFile(cmXMLWriter& xml, std::string const& file,
std::string const& name);
//! Clean test output to specified length
void CleanTestOutput(std::string& output, size_t length);

40
Source/cmCTest.cxx
View File

@@ -1612,8 +1612,33 @@ int cmCTest::GenerateDoneFile()
return 0;
}
bool cmCTest::TryToChangeDirectory(std::string const& dir)
{
cmCTestLog(this, OUTPUT,
"Internal ctest changing into directory: " << dir << std::endl);
cmsys::Status status = cmSystemTools::ChangeDirectory(dir);
if (!status) {
auto msg = "Failed to change working directory to \"" + dir +
"\" : " + status.GetString() + "\n";
cmCTestLog(this, ERROR_MESSAGE, msg);
return false;
}
return true;
}
std::string cmCTest::Base64GzipEncodeFile(std::string const& file)
{
const std::string currDir = cmSystemTools::GetCurrentWorkingDirectory();
std::string parentDir = cmSystemTools::GetParentDirectory(file);
// Temporarily change to the file's directory so the tar gets created
// with a flat directory structure.
if (currDir != parentDir) {
if (!this->TryToChangeDirectory(parentDir)) {
return "";
}
}
std::string tarFile = file + "_temp.tar.gz";
std::vector<std::string> files;
files.push_back(file);
@@ -1628,6 +1653,12 @@ std::string cmCTest::Base64GzipEncodeFile(std::string const& file)
}
std::string base64 = this->Base64EncodeFile(tarFile);
cmSystemTools::RemoveFile(tarFile);
// Change back to the directory we started in.
if (currDir != parentDir) {
cmSystemTools::ChangeDirectory(currDir);
}
return base64;
}
@@ -2853,14 +2884,7 @@ int cmCTest::ExecuteTests()
}
if (currDir != workDir) {
cmCTestLog(this, OUTPUT,
"Internal ctest changing into directory: " << workDir
<< std::endl);
cmsys::Status status = cmSystemTools::ChangeDirectory(workDir);
if (!status) {
auto msg = "Failed to change working directory to \"" + workDir +
"\" : " + status.GetString() + "\n";
cmCTestLog(this, ERROR_MESSAGE, msg);
if (!this->TryToChangeDirectory(workDir)) {
return 1;
}
}

3
Source/cmCTest.h
View File

@@ -536,6 +536,9 @@ private:
int RunCMakeAndTest(std::string* output);
int ExecuteTests();
/** return true iff change directory was successful */
bool TryToChangeDirectory(std::string const& dir);
struct Private;
std::unique_ptr<Private> Impl;
};

4
Tests/RunCMake/ctest_test/RunCMakeTest.cmake
View File

@@ -161,6 +161,10 @@ add_test(
NAME img_measurement
COMMAND ${CMAKE_COMMAND} -E
echo <DartMeasurementFile name="TestImage" type="image/png">]] ${IMAGE_DIR}/cmake-logo-16.png [[</DartMeasurementFile>)
add_test(
NAME file_measurement
COMMAND ${CMAKE_COMMAND} -E
echo <DartMeasurementFile name="my_test_input_data" type="file">]] ${IMAGE_DIR}/cmake-logo-16.png [[</DartMeasurementFile>)
]])
run_ctest(TestMeasurements)
endfunction()

5
Tests/RunCMake/ctest_test/TestMeasurements-check.cmake
View File

@@ -15,3 +15,8 @@ if(NOT _test_contents MATCHES [[NamedMeasurement name="TestImage" type="image/pn
string(APPEND RunCMake_TEST_FAILED
"Could not find expected <NamedMeasurement> tag for type='image/png' in Test.xml")
endif()
# Check file measurement.
if(NOT _test_contents MATCHES [[NamedMeasurement name="my_test_input_data" encoding="base64" compression="tar/gzip" filename="cmake-logo-16.png" type="file"]])
string(APPEND RunCMake_TEST_FAILED
"Could not find expected <NamedMeasurement> tag for type='file' in Test.xml")
endif()