AutoGen: Use depfiles for the XXX_autogen ninja targets

The XXX_autogen targets are implemented as utility commands, which
means they always run, even if there weren't any changes.

For the Ninja generator and Qt >= 5.15 we're taking a different
approach: This commit adds custom commands that create
XXX_autogen/timestamp files. Those custom commands have a depfile
assigned that is generated from the depfiles that were created by moc.

The XXX_autogen targets merely wrap the XXX_autogen/timestamp custom
commands.

Fixes: #18749

Source/cmQtAutoGenInitializer.cxx

@@ -1172,13 +1172,51 @@ bool cmQtAutoGenInitializer::InitAutogenTarget()
       }
     }
 
+    std::vector<std::string> dependencies(
+      this->AutogenTarget.DependFiles.begin(),
+      this->AutogenTarget.DependFiles.end());
+
+    const bool useNinjaDepfile = this->QtVersion >= IntegerVersion(5, 15) &&
+      this->GlobalGen->GetName().find("Ninja") != std::string::npos;
+    if (useNinjaDepfile) {
+      // Create a custom command that generates a timestamp file and
+      // has a depfile assigned. The depfile is created by JobDepFilesMergeT.
+
+      // Add additional autogen target dependencies
+      for (const cmTarget* t : this->AutogenTarget.DependTargets) {
+        dependencies.push_back(t->GetName());
+      }
+      const char timestampFileName[] = "timestamp";
+      const std::string outputFile =
+        cmStrCat(this->Dir.Build, "/", timestampFileName);
+      this->AutogenTarget.DepFile = cmStrCat(this->Dir.Build, "/deps");
+      this->AutogenTarget.DepFileRuleName =
+        cmStrCat(this->GenTarget->GetName(), "_autogen/", timestampFileName);
+      commandLines.push_back(cmMakeCommandLine(
+        { cmSystemTools::GetCMakeCommand(), "-E", "touch", outputFile }));
+
+      this->AddGeneratedSource(outputFile, this->Moc);
+      const std::string no_main_dependency;
+      this->LocalGen->AddCustomCommandToOutput(
+        outputFile, dependencies, no_main_dependency, commandLines,
+        autogenComment.c_str(), this->Dir.Work.c_str(), /*replace=*/false,
+        /*escapeOldStyle=*/false,
+        /*uses_terminal=*/false,
+        /*command_expand_lists=*/false, this->AutogenTarget.DepFile);
+
+      // Alter variables for the autogen target which now merely wraps the
+      // custom command
+      dependencies.clear();
+      dependencies.push_back(outputFile);
+      commandLines.clear();
+      autogenComment.clear();
+    }
+
     // Create autogen target
     cmTarget* autogenTarget = this->LocalGen->AddUtilityCommand(
       this->AutogenTarget.Name, true, this->Dir.Work.c_str(),
       /*byproducts=*/autogenProvides,
-      std::vector<std::string>(this->AutogenTarget.DependFiles.begin(),
+      /*depends=*/dependencies, commandLines, false, autogenComment.c_str());
-                               this->AutogenTarget.DependFiles.end()),
-      commandLines, false, autogenComment.c_str());
     // Create autogen generator target
     this->LocalGen->AddGeneratorTarget(
       cm::make_unique<cmGeneratorTarget>(autogenTarget, this->LocalGen));
@@ -1189,9 +1227,11 @@ bool cmQtAutoGenInitializer::InitAutogenTarget()
         autogenTarget->AddUtility(depName.Value, this->Makefile);
       }
     }
-    // Add additional autogen target dependencies to autogen target
+    if (!useNinjaDepfile) {
-    for (cmTarget* depTarget : this->AutogenTarget.DependTargets) {
+      // Add additional autogen target dependencies to autogen target
-      autogenTarget->AddUtility(depTarget->GetName(), this->Makefile);
+      for (cmTarget* depTarget : this->AutogenTarget.DependTargets) {
+        autogenTarget->AddUtility(depTarget->GetName(), this->Makefile);
+      }
     }
 
     // Set FOLDER property in autogen target
@@ -1416,6 +1456,8 @@ bool cmQtAutoGenInitializer::SetupWriteAutogenInfo()
   info.Set("CMAKE_EXECUTABLE", cmSystemTools::GetCMakeCommand());
   info.SetConfig("SETTINGS_FILE", this->AutogenTarget.SettingsFile);
   info.SetConfig("PARSE_CACHE_FILE", this->AutogenTarget.ParseCacheFile);
+  info.Set("DEP_FILE", this->AutogenTarget.DepFile);
+  info.Set("DEP_FILE_RULE_NAME", this->AutogenTarget.DepFileRuleName);
   info.SetArray("HEADER_EXTENSIONS",
                 this->Makefile->GetCMakeInstance()->GetHeaderExtensions());
   info.SetArrayArray(

Source/cmQtAutoGenInitializer.h

@@ -191,6 +191,8 @@ private:
     bool DependOrigin = false;
     std::set<std::string> DependFiles;
     std::set<cmTarget*> DependTargets;
+    std::string DepFile;
+    std::string DepFileRuleName;
     // Sources to process
     std::unordered_map<cmSourceFile*, MUFileHandle> Headers;
     std::unordered_map<cmSourceFile*, MUFileHandle> Sources;

Source/cmQtAutoMocUic.cxx

@@ -181,6 +181,8 @@ public:
     std::string CMakeExecutable;
     cmFileTime CMakeExecutableTime;
     std::string ParseCacheFile;
+    std::string DepFile;
+    std::string DepFileRuleName;
     std::vector<std::string> HeaderExtensions;
   };
 
@@ -516,6 +518,12 @@ public:
     void Process() override;
   };
 
+  class JobDepFilesMergeT : public JobFenceT
+  {
+  private:
+    void Process() override;
+  };
+
   /** @brief The last job.  */
   class JobFinishT : public JobFenceT
   {
@@ -1926,6 +1934,11 @@ void cmQtAutoMocUicT::JobProbeDepsFinishT::Process()
     Gen()->WorkerPool().EmplaceJob<JobMocsCompilationT>();
   }
 
+  if (!BaseConst().DepFile.empty()) {
+    // Add job to merge dep files
+    Gen()->WorkerPool().EmplaceJob<JobDepFilesMergeT>();
+  }
+
   // Add finish job
   Gen()->WorkerPool().EmplaceJob<JobFinishT>();
 }
@@ -2115,6 +2128,106 @@ void cmQtAutoMocUicT::JobMocsCompilationT::Process()
   }
 }
 
+/*
+ * Escapes paths for Ninja depfiles.
+ * This is a re-implementation of what moc does when writing depfiles.
+ */
+std::string escapeDependencyPath(cm::string_view path)
+{
+  std::string escapedPath;
+  escapedPath.reserve(path.size());
+  const size_t s = path.size();
+  int backslashCount = 0;
+  for (size_t i = 0; i < s; ++i) {
+    if (path[i] == '\\') {
+      ++backslashCount;
+    } else {
+      if (path[i] == '$') {
+        escapedPath.push_back('$');
+      } else if (path[i] == '#') {
+        escapedPath.push_back('\\');
+      } else if (path[i] == ' ') {
+        // Double the amount of written backslashes,
+        // and add one more to escape the space.
+        while (backslashCount-- >= 0) {
+          escapedPath.push_back('\\');
+        }
+      }
+      backslashCount = 0;
+    }
+    escapedPath.push_back(path[i]);
+  }
+  return escapedPath;
+}
+
+void cmQtAutoMocUicT::JobDepFilesMergeT::Process()
+{
+  if (Log().Verbose()) {
+    Log().Info(GenT::MOC, "Merging MOC dependencies");
+  }
+  auto processDepFile =
+    [](const std::string& mocOutputFile) -> std::vector<std::string> {
+    std::string f = mocOutputFile + ".d";
+    if (!cmSystemTools::FileExists(f)) {
+      return {};
+    }
+    return dependenciesFromDepFile(f.c_str());
+  };
+
+  std::vector<std::string> dependencies;
+  ParseCacheT& parseCache = BaseEval().ParseCache;
+  auto processMappingEntry = [&](const MappingMapT::value_type& m) {
+    auto cacheEntry = parseCache.GetOrInsert(m.first);
+    if (cacheEntry.first->Moc.Depends.empty()) {
+      cacheEntry.first->Moc.Depends = processDepFile(m.second->OutputFile);
+    }
+    dependencies.insert(dependencies.end(),
+                        cacheEntry.first->Moc.Depends.begin(),
+                        cacheEntry.first->Moc.Depends.end());
+  };
+
+  std::for_each(MocEval().HeaderMappings.begin(),
+                MocEval().HeaderMappings.end(), processMappingEntry);
+  std::for_each(MocEval().SourceMappings.begin(),
+                MocEval().SourceMappings.end(), processMappingEntry);
+
+  // Remove duplicates to make the depfile smaller
+  std::sort(dependencies.begin(), dependencies.end());
+  dependencies.erase(std::unique(dependencies.begin(), dependencies.end()),
+                     dependencies.end());
+
+  // Add form files
+  for (const auto& uif : UicEval().UiFiles) {
+    dependencies.push_back(uif.first);
+  }
+
+  // Write the file
+  cmsys::ofstream ofs;
+  ofs.open(BaseConst().DepFile.c_str(),
+           (std::ios::out | std::ios::binary | std::ios::trunc));
+  if (!ofs) {
+    LogError(GenT::GEN,
+             cmStrCat("Cannot open ", MessagePath(BaseConst().DepFile),
+                      " for writing."));
+    return;
+  }
+  ofs << BaseConst().DepFileRuleName << ": \\" << std::endl;
+  for (const std::string& file : dependencies) {
+    ofs << '\t' << escapeDependencyPath(file) << " \\" << std::endl;
+    if (!ofs.good()) {
+      LogError(GenT::GEN,
+               cmStrCat("Writing depfile", MessagePath(BaseConst().DepFile),
+                        " failed."));
+      return;
+    }
+  }
+
+  // Add the CMake executable to re-new cache data if necessary.
+  // Also, this is the last entry, so don't add a backslash.
+  ofs << '\t' << escapeDependencyPath(BaseConst().CMakeExecutable)
+      << std::endl;
+}
+
 void cmQtAutoMocUicT::JobFinishT::Process()
 {
   Gen()->AbortSuccess();
@@ -2139,6 +2252,9 @@ bool cmQtAutoMocUicT::InitFromInfo(InfoT const& info)
       !info.GetString("CMAKE_EXECUTABLE", BaseConst_.CMakeExecutable, true) ||
       !info.GetStringConfig("PARSE_CACHE_FILE", BaseConst_.ParseCacheFile,
                             true) ||
+      !info.GetString("DEP_FILE", BaseConst_.DepFile, false) ||
+      !info.GetString("DEP_FILE_RULE_NAME", BaseConst_.DepFileRuleName,
+                      false) ||
       !info.GetStringConfig("SETTINGS_FILE", SettingsFile_, true) ||
       !info.GetArray("HEADER_EXTENSIONS", BaseConst_.HeaderExtensions, true) ||
       !info.GetString("QT_MOC_EXECUTABLE", MocConst_.Executable, false) ||

Tests/RunCMake/NinjaMultiConfig/Qt5.cmake

@@ -17,6 +17,10 @@ if(Qt5Core_VERSION VERSION_GREATER_EQUAL "5.15.0")
 endif()
 
 set(autogen_files "${CMAKE_BINARY_DIR}/exe_autogen/mocs_compilation.cpp")
+if(moc_writes_depfiles)
+  list(APPEND autogen_files "${CMAKE_BINARY_DIR}/exe_autogen/deps")
+  list(APPEND autogen_files "${CMAKE_BINARY_DIR}/exe_autogen/timestamp")
+endif()
 foreach(c IN LISTS CMAKE_CONFIGURATION_TYPES)
   list(APPEND autogen_files "${CMAKE_BINARY_DIR}/exe_autogen/include_${c}/moc_qt5.cpp")
   if(moc_writes_depfiles)