Kitware/CMake
1
0
Fork 0
mirror of https://github.com/Kitware/CMake.git synced 2025-10-19 02:17:27 +08:00
Code Issues Projects Releases Wiki Activity

install: Write manifest file in parallel install

Browse Source 
Updates the parallel install to generate the install_manifest.txt file.
This commit is contained in:
Martin Duffy
2024-07-18 14:57:40 -04:00
parent d39b8044b6
commit 9799491c7e
20 changed files with 225 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
Source/cmInstallScriptHandler.cxx
View File

@@ -16,6 +16,11 @@
#include <cm3p/json/value.h>
#include <cm3p/uv.h>
#include "cmsys/FStream.hxx"
#include "cmsys/RegularExpression.hxx"
#include "cmCryptoHash.h"
#include "cmGeneratedFileStream.h"
#include "cmJSONState.h"
#include "cmProcessOutput.h"
#include "cmStringAlgorithms.h"
@@ -26,15 +31,19 @@
using InstallScript = cmInstallScriptHandler::InstallScript;
cmInstallScriptHandler::cmInstallScriptHandler(const std::string& binary_dir,
cmInstallScriptHandler::cmInstallScriptHandler(std::string _binaryDir,
std::string _component,
std::vector<std::string>& args)
: binaryDir(std::move(_binaryDir))
, component(std::move(_component))
{
const std::string& file =
cmStrCat(binary_dir, "/CMakeFiles/InstallScripts.json");
cmStrCat(this->binaryDir, "/CMakeFiles/InstallScripts.json");
if (cmSystemTools::FileExists(file)) {
int compare;
cmSystemTools::FileTimeCompare(
cmStrCat(binary_dir, "/CMakeFiles/cmake.check_cache"), file, &compare);
cmStrCat(this->binaryDir, "/CMakeFiles/cmake.check_cache"), file,
&compare);
if (compare < 1) {
args.insert(args.end() - 1, "-DCMAKE_INSTALL_LOCAL_ONLY=1");
Json::CharReaderBuilder rbuilder;
@@ -46,6 +55,8 @@ cmInstallScriptHandler::cmInstallScriptHandler(const std::string& binary_dir,
for (auto const& script : value["InstallScripts"]) {
this->commands.push_back(args);
this->commands.back().emplace_back(script.asCString());
this->directories.push_back(
cmSystemTools::GetFilenamePath(script.asCString()));
}
}
}
@@ -79,6 +90,36 @@ int cmInstallScriptHandler::install(unsigned int j)
}
uv_run(loop, UV_RUN_DEFAULT);
}
// Write install manifest
std::string install_manifest;
if (this->component.empty()) {
install_manifest = "install_manifest.txt";
} else {
cmsys::RegularExpression regEntry;
if (regEntry.compile("^[a-zA-Z0-9_.+-]+$") &&
regEntry.find(this->component)) {
install_manifest =
cmStrCat("install_manifest_", this->component, ".txt");
} else {
cmCryptoHash md5(cmCryptoHash::AlgoMD5);
md5.Initialize();
install_manifest =
cmStrCat("install_manifest_", md5.HashString(this->component), ".txt");
}
}
cmGeneratedFileStream fout(cmStrCat(this->binaryDir, "/", install_manifest));
fout.SetCopyIfDifferent(true);
for (auto const& dir : this->directories) {
auto local_manifest = cmStrCat(dir, "/install_local_manifest.txt");
if (cmSystemTools::FileExists(local_manifest)) {
cmsys::ifstream fin(local_manifest.c_str());
std::string line;
while (std::getline(fin, line)) {
fout << line << "\n";
}
}
}
return 0;
}

5
Source/cmInstallScriptHandler.h
View File

@@ -16,7 +16,7 @@ class cmInstallScriptHandler
{
public:
cmInstallScriptHandler() = default;
cmInstallScriptHandler(const std::string&, std::vector<std::string>&);
cmInstallScriptHandler(std::string, std::string, std::vector<std::string>&);
bool isParallel();
int install(unsigned int j);
class InstallScript
@@ -37,4 +37,7 @@ public:
private:
std::vector<std::vector<std::string>> commands;
std::vector<std::string> directories;
std::string binaryDir;
std::string component;
};

52
Source/cmLocalGenerator.cxx
View File

@@ -713,32 +713,40 @@ void cmLocalGenerator::GenerateInstallRules()
break;
}
// Record the install manifest.
if (toplevel_install) {
/* clang-format off */
/* clang-format off */
fout <<
"if(CMAKE_INSTALL_COMPONENT)\n"
" if(CMAKE_INSTALL_COMPONENT MATCHES \"^[a-zA-Z0-9_.+-]+$\")\n"
" set(CMAKE_INSTALL_MANIFEST \"install_manifest_"
"${CMAKE_INSTALL_COMPONENT}.txt\")\n"
" else()\n"
" string(MD5 CMAKE_INST_COMP_HASH \"${CMAKE_INSTALL_COMPONENT}\")\n"
" set(CMAKE_INSTALL_MANIFEST \"install_manifest_"
"${CMAKE_INST_COMP_HASH}.txt\")\n"
" unset(CMAKE_INST_COMP_HASH)\n"
" endif()\n"
"else()\n"
" set(CMAKE_INSTALL_MANIFEST \"install_manifest.txt\")\n"
"endif()\n"
"\n"
"if(NOT CMAKE_INSTALL_LOCAL_ONLY)\n"
" string(REPLACE \";\" \"\\n\" CMAKE_INSTALL_MANIFEST_CONTENT\n"
"string(REPLACE \";\" \"\\n\" CMAKE_INSTALL_MANIFEST_CONTENT\n"
" \"${CMAKE_INSTALL_MANIFEST_FILES}\")\n"
" file(WRITE \"" << homedir << "/${CMAKE_INSTALL_MANIFEST}\"\n"
"if(CMAKE_INSTALL_LOCAL_ONLY)\n"
" file(WRITE \"" <<
this->StateSnapshot.GetDirectory().GetCurrentBinary() <<
"/install_local_manifest.txt\"\n"
" \"${CMAKE_INSTALL_MANIFEST_CONTENT}\")\n"
"endif()\n";
/* clang-format on */
}
if (toplevel_install) {
fout <<
"if(CMAKE_INSTALL_COMPONENT)\n"
" if(CMAKE_INSTALL_COMPONENT MATCHES \"^[a-zA-Z0-9_.+-]+$\")\n"
" set(CMAKE_INSTALL_MANIFEST \"install_manifest_"
"${CMAKE_INSTALL_COMPONENT}.txt\")\n"
" else()\n"
" string(MD5 CMAKE_INST_COMP_HASH \"${CMAKE_INSTALL_COMPONENT}\")\n"
" set(CMAKE_INSTALL_MANIFEST \"install_manifest_"
"${CMAKE_INST_COMP_HASH}.txt\")\n"
" unset(CMAKE_INST_COMP_HASH)\n"
" endif()\n"
"else()\n"
" set(CMAKE_INSTALL_MANIFEST \"install_manifest.txt\")\n"
"endif()\n"
"\n"
"if(NOT CMAKE_INSTALL_LOCAL_ONLY)\n"
" file(WRITE \"" << homedir << "/${CMAKE_INSTALL_MANIFEST}\"\n"
" \"${CMAKE_INSTALL_MANIFEST_CONTENT}\")\n"
"endif()\n";
}
/* clang-format on */
}
void cmLocalGenerator::AddGeneratorTarget(

2
Source/cmakemain.cxx
View File

@@ -922,7 +922,7 @@ int do_install(int ac, char const* const* av)
args.emplace_back("-P");
auto handler = cmInstallScriptHandler(dir, args);
auto handler = cmInstallScriptHandler(dir, component, args);
int ret = 0;
if (!handler.isParallel()) {
args.emplace_back(cmStrCat(dir, "/cmake_install.cmake"));

53
Tests/RunCMake/InstallParallel/RunCMakeTest.cmake
View File

@@ -1,36 +1,59 @@
include(RunCMake)
function(install_test test parallel install_arg)
cmake_parse_arguments(ARGS "NINJA;TOUCH_CACHE" "VERIFY_SCRIPT" "" ${ARGN})
function(install_test test)
cmake_parse_arguments(ARGS "PARALLEL;NINJA;TOUCH_CACHE" "ARGS;COMPONENT" "" ${ARGN})
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/${test}-install)
set(RunCMake_TEST_OPTIONS -DINSTALL_PARALLEL=${parallel})
set(RunCMake_TEST_OPTIONS -DINSTALL_PARALLEL=${ARGS_PARALLEL} -DCMAKE_INSTALL_PREFIX=install)
set(RunCMake_TEST_OUTPUT_MERGE 1)
if (NOT RunCMake_GENERATOR_IS_MULTI_CONFIG)
list(APPEND RunCMake_TEST_OPTIONS -DCMAKE_BUILD_TYPE=Debug)
endif()
if (ARGS_COMPONENT)
list(APPEND ARGS_ARGS "--component ${ARGS_COMPONENT}")
endif()
run_cmake(install)
set(RunCMake_TEST_NO_CLEAN 1)
if (ARGS_TOUCH_CACHE)
run_cmake_command(${test}-sleep ${CMAKE_COMMAND} -E sleep 2)
run_cmake_command(${test}-touch
${CMAKE_COMMAND} -E touch ${RunCMake_TEST_BINARY_DIR}/CMakeFiles/cmake.check_cache)
endif()
if (ARGS_NINJA)
run_cmake_command(${test}-install ${CMAKE_COMMAND} --build . --config Debug -t ${install_arg})
if (ARGS_PARALLEL)
set(INSTALL_COUNT 5)
else()
set(INSTALL_COUNT 1)
endif()
set(RunCMake-check-file check-num-installs.cmake)
run_cmake_command(${test}-install ${CMAKE_COMMAND} --build . --config Debug ${ARGS_ARGS})
unset(RunCMake-check-file)
else()
run_cmake_command(${test}-install ${CMAKE_COMMAND} --install . -j ${install_arg})
if (ARGS_COMPONENT)
if(ARGS_COMPONENT MATCHES "^[a-zA-Z0-9_.+-]+$")
set(INSTALL_MANIFEST "install_manifest_${ARGS_COMPONENT}.txt")
else()
string(MD5 COMPONENT_HASH "${ARGS_COMPONENT}")
set(INSTALL_MANIFEST "install_manifest_${COMPONENT_HASH}.txt")
endif()
set(INSTALL_COUNT 0)
else()
set(INSTALL_MANIFEST "install_manifest.txt")
set(INSTALL_COUNT 5)
endif()
set(INSTALL_MANIFEST ${RunCMake_TEST_BINARY_DIR}/${INSTALL_MANIFEST})
set(RunCMake-check-file check-manifest.cmake)
run_cmake_command(${test}-install ${CMAKE_COMMAND} --install . ${ARGS_ARGS})
unset(RunCMake-check-file)
endif()
set(RunCMake_TEST_COMMAND_WORKING_DIRECTORY ${RunCMake_SOURCE_DIR})
if (ARGS_VERIFY_SCRIPT)
run_cmake_command(${test}-verify-parallel
${CMAKE_COMMAND} -P ${ARGS_VERIFY_SCRIPT} ${RunCMake_TEST_BINARY_DIR}/.ninja_log)
endif()
endfunction()
install_test(parallel 1 4)
install_test(no-parallel 0 4)
install_test(out-of-date-json 1 4 TOUCH_CACHE)
install_test(parallel PARALLEL ARGS "-j 4")
install_test(no-parallel ARGS "-j 4")
install_test(out-of-date-json TOUCH_CACHE PARALLEL ARGS "-j 4")
install_test(component PARALLEL ARGS "-j 4" COMPONENT "ALPHANUMERIC123")
install_test(component-hash PARALLEL ARGS "-j 4" COMPONENT "@#$")
if(RunCMake_GENERATOR MATCHES "Ninja")
install_test(ninja-parallel 1 install/parallel VERIFY_SCRIPT check-parallel.cmake NINJA)
install_test(ninja-no-parallel 0 install VERIFY_SCRIPT check-single.cmake NINJA)
install_test(ninja-parallel ARGS "-t install/parallel" NINJA PARALLEL)
install_test(ninja-no-parallel ARGS "-t install" NINJA)
endif()

10
Tests/RunCMake/InstallParallel/check-manifest.cmake Normal file
View File

@@ -0,0 +1,10 @@
if (NOT EXISTS ${INSTALL_MANIFEST})
set(RunCMake_TEST_FAILED "Install manifest not generated: ${INSTALL_MANIFEST}")
endif()
file(STRINGS ${INSTALL_MANIFEST} lines ENCODING UTF-8)
list(LENGTH lines len)
if (NOT len EQUAL ${INSTALL_COUNT})
set(RunCMake_TEST_FAILED "Install manifest missing content: ${len}/${INSTALL_COUNT}")
endif()

7
Tests/RunCMake/InstallParallel/check-num-installs.cmake Normal file
View File

@@ -0,0 +1,7 @@
file(STRINGS ${RunCMake_TEST_BINARY_DIR}/.ninja_log lines ENCODING UTF-8)
list(FILTER lines INCLUDE REGEX ".*install.*util")
list(LENGTH lines len)
if (NOT ${len} STREQUAL ${INSTALL_COUNT})
set(RunCMake_TEST_FAILED "Wrong number of cmake -P calls to install: ${len}/${INSTALL_COUNT}")
endif()

15
Tests/RunCMake/InstallParallel/check-parallel.cmake
View File

@@ -1,15 +0,0 @@
include(read-ninja-install.cmake)
foreach(line ${lines})
string(REPLACE "\t" ";" line ${line})
list(GET line 0 start)
list(GET line 1 end)
list(APPEND start_times ${start})
list(APPEND end_times ${end})
endforeach()
list(GET start_times 1 start_2)
list(GET end_times 0 end_1)
if (NOT start_2 LESS end_1)
message(FATAL_ERROR "Install is not parallel")
endif()

5
Tests/RunCMake/InstallParallel/check-single.cmake
View File

@@ -1,5 +0,0 @@
include(read-ninja-install.cmake)
list(LENGTH lines len)
if (NOT ${len} STREQUAL "1")
message(FATAL_ERROR "Expected single installation call")
endif()

2
Tests/RunCMake/InstallParallel/install.cmake
View File

@@ -1,4 +1,4 @@
install(CODE "message(installing:${CMAKE_CURRENT_SOURCE_DIR})")
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt TYPE DATA RENAME root.txt)
if (INSTALL_PARALLEL)
set_property(GLOBAL PROPERTY INSTALL_PARALLEL ON)
endif()

17
Tests/RunCMake/InstallParallel/ninja-no-parallel-install-stdout.txt
View File

@@ -1,5 +1,12 @@
installing:.*
installing:.*
installing:.*
installing:.*
installing:.*
\-\- Install configuration:[^
]*
\-\- Installing:[^
]*
\-\- Installing:[^
]*
\-\- Installing:[^
]*
\-\- Installing:[^
]*
\-\- Installing:[^
]*

30
Tests/RunCMake/InstallParallel/ninja-parallel-install-stdout.txt
View File

@@ -1,15 +1,25 @@
\[1\/5\] Installing only the local directory...
\-\- Install configuration:.*
installing:.*
\-\- Install configuration:[^
]*
\-\- Installing:[^
]*
\[2\/5\] Installing only the local directory...
\-\- Install configuration:.*
installing:.*
\-\- Install configuration:[^
]*
\-\- Installing:[^
]*
\[3\/5\] Installing only the local directory...
\-\- Install configuration:.*
installing:.*
\-\- Install configuration:[^
]*
\-\- Installing:[^
]*
\[4\/5\] Installing only the local directory...
\-\- Install configuration:.*
installing:.*
\-\- Install configuration:[^
]*
\-\- Installing:[^
]*
\[5\/5\] Installing only the local directory...
\-\- Install configuration:.*
installing:.*
\-\- Install configuration:[^
]*
\-\- Installing:[^
]*

17
Tests/RunCMake/InstallParallel/no-parallel-install-stdout.txt
View File

@@ -1,5 +1,12 @@
installing:.*
installing:.*
installing:.*
installing:.*
installing:.*
\-\- Install configuration:[^
]*
\-\- Installing:[^
]*
\-\- Installing:[^
]*
\-\- Installing:[^
]*
\-\- Installing:[^
]*
\-\- Installing:[^
]*

17
Tests/RunCMake/InstallParallel/out-of-date-json-install-stdout.txt
View File

@@ -1,5 +1,12 @@
installing:.*
installing:.*
installing:.*
installing:.*
installing:.*
\-\- Install configuration:[^
]*
\-\- Installing:[^
]*
\-\- Installing:[^
]*
\-\- Installing:[^
]*
\-\- Installing:[^
]*
\-\- Installing:[^
]*

45
Tests/RunCMake/InstallParallel/parallel-install-stdout.txt
View File

@@ -1,15 +1,30 @@
\[1\/5\] .*
\-\- Install configuration:.*
installing:.*
\[2\/5\] .*
\-\- Install configuration:.*
installing:.*
\[3\/5\] .*
\-\- Install configuration:.*
installing:.*
\[4\/5\] .*
\-\- Install configuration:.*
installing:.*
\[5\/5\] .*
\-\- Install configuration:.*
installing:.*
\[1\/5\] [^
]*
\-\- Install configuration:[^
]*
\-\- Installing:[^
]*
\[2\/5\] [^
]*
\-\- Install configuration:[^
]*
\-\- Installing:[^
]*
\[3\/5\] [^
]*
\-\- Install configuration:[^
]*
\-\- Installing:[^
]*
\[4\/5\] [^
]*
\-\- Install configuration:[^
]*
\-\- Installing:[^
]*
\[5\/5\] [^
]*
\-\- Install configuration:[^
]*
\-\- Installing:[^
]*

4
Tests/RunCMake/InstallParallel/read-ninja-install.cmake
View File

@@ -1,4 +0,0 @@
set(ninja_log ${CMAKE_ARGV3})
file(STRINGS ${ninja_log} lines)
list(POP_FRONT lines)
list(FILTER lines INCLUDE REGEX ".*install.*util")

2
Tests/RunCMake/InstallParallel/subdir-1/CMakeLists.txt
View File

@@ -1,3 +1,3 @@
install(CODE "message(installing:${CMAKE_CURRENT_SOURCE_DIR})")
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt TYPE DATA RENAME 1.txt)
add_subdirectory(subdir-3)
add_subdirectory(subdir-4)

2
Tests/RunCMake/InstallParallel/subdir-1/subdir-3/CMakeLists.txt
View File

@@ -1 +1 @@
install(CODE "message(installing:${CMAKE_CURRENT_SOURCE_DIR})")
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt TYPE DATA RENAME 3.txt)

2
Tests/RunCMake/InstallParallel/subdir-1/subdir-4/CMakeLists.txt
View File

@@ -1 +1 @@
install(CODE "message(installing:${CMAKE_CURRENT_SOURCE_DIR})")
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt TYPE DATA RENAME 4.txt)

2
Tests/RunCMake/InstallParallel/subdir-2/CMakeLists.txt
View File

@@ -1 +1 @@
install(CODE "message(installing:${CMAKE_CURRENT_SOURCE_DIR})")
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt TYPE DATA RENAME 2.txt)