Windows: Encode child process output to internally-used encoding

Typically Windows applications (eg. MSVC compiler) use current console's
codepage for output to pipes so we need to encode that to our
internally-used encoding (`KWSYS_ENCODING_DEFAULT_CODEPAGE`).

Source/CTest/cmCTestBuildHandler.cxx

@@ -7,6 +7,7 @@
 #include "cmFileTimeComparison.h"
 #include "cmGeneratedFileStream.h"
 #include "cmMakefile.h"
+#include "cmProcessOutput.h"
 #include "cmSystemTools.h"
 #include "cmXMLWriter.h"
 
@@ -809,6 +810,8 @@ int cmCTestBuildHandler::RunMakeCommand(const char* command, int* retVal,
 
   char* data;
   int length;
+  cmProcessOutput processOutput;
+  std::string strdata;
   cmCTestOptionalLog(
     this->CTest, HANDLER_PROGRESS_OUTPUT, "   Each symbol represents "
       << tick_len << " bytes of output." << std::endl
@@ -842,13 +845,25 @@ int cmCTestBuildHandler::RunMakeCommand(const char* command, int* retVal,
 
     // Process the chunk of data
     if (res == cmsysProcess_Pipe_STDERR) {
-      this->ProcessBuffer(data, length, tick, tick_len, ofs,
+      processOutput.DecodeText(data, length, strdata, 1);
+      this->ProcessBuffer(strdata.c_str(), strdata.size(), tick, tick_len, ofs,
                           &this->BuildProcessingErrorQueue);
     } else {
-      this->ProcessBuffer(data, length, tick, tick_len, ofs,
+      processOutput.DecodeText(data, length, strdata, 2);
+      this->ProcessBuffer(strdata.c_str(), strdata.size(), tick, tick_len, ofs,
                           &this->BuildProcessingQueue);
     }
   }
+  processOutput.DecodeText(std::string(), strdata, 1);
+  if (!strdata.empty()) {
+    this->ProcessBuffer(strdata.c_str(), strdata.size(), tick, tick_len, ofs,
+                        &this->BuildProcessingErrorQueue);
+  }
+  processOutput.DecodeText(std::string(), strdata, 2);
+  if (!strdata.empty()) {
+    this->ProcessBuffer(strdata.c_str(), strdata.size(), tick, tick_len, ofs,
+                        &this->BuildProcessingQueue);
+  }
 
   this->ProcessBuffer(CM_NULLPTR, 0, tick, tick_len, ofs,
                       &this->BuildProcessingQueue);

Source/CTest/cmCTestLaunch.cxx

@@ -8,6 +8,7 @@
 #include "cmGeneratedFileStream.h"
 #include "cmGlobalGenerator.h"
 #include "cmMakefile.h"
+#include "cmProcessOutput.h"
 #include "cmStateTypes.h"
 #include "cmSystemTools.h"
 #include "cmXMLWriter.h"
@@ -225,17 +226,31 @@ void cmCTestLaunch::RunChild()
   if (!this->Passthru) {
     char* data = CM_NULLPTR;
     int length = 0;
+    cmProcessOutput processOutput;
+    std::string strdata;
     while (int p = cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR)) {
       if (p == cmsysProcess_Pipe_STDOUT) {
-        fout.write(data, length);
-        std::cout.write(data, length);
+        processOutput.DecodeText(data, length, strdata, 1);
+        fout.write(strdata.c_str(), strdata.size());
+        std::cout.write(strdata.c_str(), strdata.size());
         this->HaveOut = true;
       } else if (p == cmsysProcess_Pipe_STDERR) {
-        ferr.write(data, length);
-        std::cerr.write(data, length);
+        processOutput.DecodeText(data, length, strdata, 2);
+        ferr.write(strdata.c_str(), strdata.size());
+        std::cerr.write(strdata.c_str(), strdata.size());
         this->HaveErr = true;
       }
     }
+    processOutput.DecodeText(std::string(), strdata, 1);
+    if (!strdata.empty()) {
+      fout.write(strdata.c_str(), strdata.size());
+      std::cout.write(strdata.c_str(), strdata.size());
+    }
+    processOutput.DecodeText(std::string(), strdata, 2);
+    if (!strdata.empty()) {
+      ferr.write(strdata.c_str(), strdata.size());
+      std::cerr.write(strdata.c_str(), strdata.size());
+    }
   }
 
   // Wait for the real command to finish.

Source/CTest/cmCTestSubmitHandler.cxx

@@ -7,6 +7,7 @@
 #include "cmCTestScriptHandler.h"
 #include "cmCurl.h"
 #include "cmGeneratedFileStream.h"
+#include "cmProcessOutput.h"
 #include "cmState.h"
 #include "cmStateTypes.h"
 #include "cmSystemTools.h"
@@ -784,10 +785,20 @@ bool cmCTestSubmitHandler::SubmitUsingSCP(const std::string& scp_command,
     cmsysProcess_Execute(cp);
     char* data;
     int length;
+    cmProcessOutput processOutput;
+    std::string strdata;
 
     while (cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR)) {
+      processOutput.DecodeText(data, length, strdata);
       cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
-                         cmCTestLogWrite(data, length), this->Quiet);
+                         cmCTestLogWrite(strdata.c_str(), strdata.size()),
+                         this->Quiet);
+    }
+    processOutput.DecodeText(std::string(), strdata);
+    if (!strdata.empty()) {
+      cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
+                         cmCTestLogWrite(strdata.c_str(), strdata.size()),
+                         this->Quiet);
     }
 
     cmsysProcess_WaitForExit(cp, CM_NULLPTR);

Source/CTest/cmProcess.cxx

@@ -3,6 +3,7 @@
 #include "cmProcess.h"
 
 #include <cmConfigure.h>
+#include <cmProcessOutput.h>
 #include <cmSystemTools.h>
 #include <iostream>
 
@@ -104,6 +105,8 @@ bool cmProcess::Buffer::GetLast(std::string& line)
 
 int cmProcess::GetNextOutputLine(std::string& line, double timeout)
 {
+  cmProcessOutput processOutput;
+  std::string strdata;
   for (;;) {
     // Look for lines already buffered.
     if (this->Output.GetLine(line)) {
@@ -118,12 +121,17 @@ int cmProcess::GetNextOutputLine(std::string& line, double timeout)
       return cmsysProcess_Pipe_Timeout;
     }
     if (p == cmsysProcess_Pipe_STDOUT) {
-      this->Output.insert(this->Output.end(), data, data + length);
+      processOutput.DecodeText(data, length, strdata);
+      this->Output.insert(this->Output.end(), strdata.begin(), strdata.end());
     } else { // p == cmsysProcess_Pipe_None
       // The process will provide no more data.
       break;
     }
   }
+  processOutput.DecodeText(std::string(), strdata);
+  if (!strdata.empty()) {
+    this->Output.insert(this->Output.end(), strdata.begin(), strdata.end());
+  }
 
   // Look for partial last lines.
   if (this->Output.GetLast(line)) {

Source/cmCTest.cxx

@@ -41,6 +41,7 @@
 #include "cmGeneratedFileStream.h"
 #include "cmGlobalGenerator.h"
 #include "cmMakefile.h"
+#include "cmProcessOutput.h"
 #include "cmState.h"
 #include "cmStateSnapshot.h"
 #include "cmStateTypes.h"
@@ -999,16 +1000,19 @@ int cmCTest::RunMakeCommand(const char* command, std::string& output,
 
   char* data;
   int length;
+  cmProcessOutput processOutput;
+  std::string strdata;
   cmCTestLog(this, HANDLER_PROGRESS_OUTPUT, "   Each . represents "
                << tick_len << " bytes of output" << std::endl
                << "    " << std::flush);
   while (cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR)) {
-    for (int cc = 0; cc < length; ++cc) {
-      if (data[cc] == 0) {
-        data[cc] = '\n';
+    processOutput.DecodeText(data, length, strdata);
+    for (size_t cc = 0; cc < strdata.size(); ++cc) {
+      if (strdata[cc] == 0) {
+        strdata[cc] = '\n';
       }
     }
-    output.append(data, length);
+    output.append(strdata);
     while (output.size() > (tick * tick_len)) {
       tick++;
       cmCTestLog(this, HANDLER_PROGRESS_OUTPUT, "." << std::flush);
@@ -1019,9 +1023,19 @@ int cmCTest::RunMakeCommand(const char* command, std::string& output,
                               << "    " << std::flush);
       }
     }
-    cmCTestLog(this, HANDLER_VERBOSE_OUTPUT, cmCTestLogWrite(data, length));
+    cmCTestLog(this, HANDLER_VERBOSE_OUTPUT,
+               cmCTestLogWrite(strdata.c_str(), strdata.size()));
     if (ofs) {
-      ofs << cmCTestLogWrite(data, length);
+      ofs << cmCTestLogWrite(strdata.c_str(), strdata.size());
+    }
+  }
+  processOutput.DecodeText(std::string(), strdata);
+  if (!strdata.empty()) {
+    output.append(strdata);
+    cmCTestLog(this, HANDLER_VERBOSE_OUTPUT,
+               cmCTestLogWrite(strdata.c_str(), strdata.size()));
+    if (ofs) {
+      ofs << cmCTestLogWrite(strdata.c_str(), strdata.size());
     }
   }
   cmCTestLog(this, HANDLER_PROGRESS_OUTPUT, " Size of output: "
@@ -1156,17 +1170,30 @@ int cmCTest::RunTest(std::vector<const char*> argv, std::string* output,
 
   char* data;
   int length;
+  cmProcessOutput processOutput;
+  std::string strdata;
   while (cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR)) {
+    processOutput.DecodeText(data, length, strdata);
     if (output) {
       tempOutput.insert(tempOutput.end(), data, data + length);
     }
-    cmCTestLog(this, HANDLER_VERBOSE_OUTPUT, cmCTestLogWrite(data, length));
+    cmCTestLog(this, HANDLER_VERBOSE_OUTPUT,
+               cmCTestLogWrite(strdata.c_str(), strdata.size()));
     if (log) {
-      log->write(data, length);
+      log->write(strdata.c_str(), strdata.size());
+    }
+  }
+  processOutput.DecodeText(std::string(), strdata);
+  if (!strdata.empty()) {
+    cmCTestLog(this, HANDLER_VERBOSE_OUTPUT,
+               cmCTestLogWrite(strdata.c_str(), strdata.size()));
+    if (log) {
+      log->write(strdata.c_str(), strdata.size());
     }
   }
 
   cmsysProcess_WaitForExit(cp, CM_NULLPTR);
+  processOutput.DecodeText(tempOutput, tempOutput);
   if (output && tempOutput.begin() != tempOutput.end()) {
     output->append(&*tempOutput.begin(), tempOutput.size());
   }
@@ -2527,6 +2554,8 @@ bool cmCTest::RunCommand(const char* command, std::string* stdOut,
   std::vector<char> tempError;
   char* data;
   int length;
+  cmProcessOutput processOutput;
+  std::string strdata;
   int res;
   bool done = false;
   while (!done) {
@@ -2543,15 +2572,24 @@ bool cmCTest::RunCommand(const char* command, std::string* stdOut,
     }
     if ((res == cmsysProcess_Pipe_STDOUT || res == cmsysProcess_Pipe_STDERR) &&
         this->ExtraVerbose) {
-      cmSystemTools::Stdout(data, length);
+      processOutput.DecodeText(data, length, strdata);
+      cmSystemTools::Stdout(strdata.c_str(), strdata.size());
+    }
+  }
+  if (this->ExtraVerbose) {
+    processOutput.DecodeText(std::string(), strdata);
+    if (!strdata.empty()) {
+      cmSystemTools::Stdout(strdata.c_str(), strdata.size());
     }
   }
 
   cmsysProcess_WaitForExit(cp, CM_NULLPTR);
   if (!tempOutput.empty()) {
+    processOutput.DecodeText(tempOutput, tempOutput);
     stdOut->append(&*tempOutput.begin(), tempOutput.size());
   }
   if (!tempError.empty()) {
+    processOutput.DecodeText(tempError, tempError);
     stdErr->append(&*tempError.begin(), tempError.size());
   }
 

Source/cmExecProgramCommand.cxx

@@ -6,6 +6,7 @@
 #include <stdio.h>
 
 #include "cmMakefile.h"
+#include "cmProcessOutput.h"
 #include "cmSystemTools.h"
 
 class cmExecutionStatus;
@@ -214,17 +215,28 @@ bool cmExecProgramCommand::RunCommand(const char* command, std::string& output,
   int length;
   char* data;
   int p;
+  cmProcessOutput processOutput;
+  std::string strdata;
   while ((p = cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR), p)) {
     if (p == cmsysProcess_Pipe_STDOUT || p == cmsysProcess_Pipe_STDERR) {
       if (verbose) {
-        cmSystemTools::Stdout(data, length);
+        processOutput.DecodeText(data, length, strdata);
+        cmSystemTools::Stdout(strdata.c_str(), strdata.size());
       }
       output.append(data, length);
     }
   }
 
+  if (verbose) {
+    processOutput.DecodeText(std::string(), strdata);
+    if (!strdata.empty()) {
+      cmSystemTools::Stdout(strdata.c_str(), strdata.size());
+    }
+  }
+
   // All output has been read.  Wait for the process to exit.
   cmsysProcess_WaitForExit(cp, CM_NULLPTR);
+  processOutput.DecodeText(output, output);
 
   // Check the result of running the process.
   std::string msg;

Source/cmExecuteProcessCommand.cxx

@@ -8,6 +8,7 @@
 #include <stdio.h>
 
 #include "cmMakefile.h"
+#include "cmProcessOutput.h"
 #include "cmSystemTools.h"
 
 class cmExecutionStatus;
@@ -222,25 +223,43 @@ bool cmExecuteProcessCommand::InitialPass(std::vector<std::string> const& args,
   int length;
   char* data;
   int p;
+  cmProcessOutput processOutput;
+  std::string strdata;
   while ((p = cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR), p)) {
     // Put the output in the right place.
     if (p == cmsysProcess_Pipe_STDOUT && !output_quiet) {
       if (output_variable.empty()) {
-        cmSystemTools::Stdout(data, length);
+        processOutput.DecodeText(data, length, strdata, 1);
+        cmSystemTools::Stdout(strdata.c_str(), strdata.size());
       } else {
         cmExecuteProcessCommandAppend(tempOutput, data, length);
       }
     } else if (p == cmsysProcess_Pipe_STDERR && !error_quiet) {
       if (error_variable.empty()) {
-        cmSystemTools::Stderr(data, length);
+        processOutput.DecodeText(data, length, strdata, 2);
+        cmSystemTools::Stderr(strdata.c_str(), strdata.size());
       } else {
         cmExecuteProcessCommandAppend(tempError, data, length);
       }
     }
   }
+  if (!output_quiet && output_variable.empty()) {
+    processOutput.DecodeText(std::string(), strdata, 1);
+    if (!strdata.empty()) {
+      cmSystemTools::Stdout(strdata.c_str(), strdata.size());
+    }
+  }
+  if (!error_quiet && error_variable.empty()) {
+    processOutput.DecodeText(std::string(), strdata, 2);
+    if (!strdata.empty()) {
+      cmSystemTools::Stderr(strdata.c_str(), strdata.size());
+    }
+  }
 
   // All output has been read.  Wait for the process to exit.
   cmsysProcess_WaitForExit(cp, CM_NULLPTR);
+  processOutput.DecodeText(tempOutput, tempOutput);
+  processOutput.DecodeText(tempError, tempError);
 
   // Fix the text in the output strings.
   cmExecuteProcessCommandFixText(tempOutput, output_strip_trailing_whitespace);

Source/cmProcessTools.cxx

@@ -1,6 +1,7 @@
 /* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
    file Copyright.txt or https://cmake.org/licensing for details.  */
 #include "cmProcessTools.h"
+#include "cmProcessOutput.h"
 
 #include <cmsys/Process.h>
 #include <ostream>
@@ -12,18 +13,34 @@ void cmProcessTools::RunProcess(struct cmsysProcess_s* cp, OutputParser* out,
   char* data = CM_NULLPTR;
   int length = 0;
   int p;
+  cmProcessOutput processOutput;
+  std::string strdata;
   while ((out || err) &&
          (p = cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR), p)) {
     if (out && p == cmsysProcess_Pipe_STDOUT) {
-      if (!out->Process(data, length)) {
+      processOutput.DecodeText(data, length, strdata, 1);
+      if (!out->Process(strdata.c_str(), int(strdata.size()))) {
         out = CM_NULLPTR;
       }
     } else if (err && p == cmsysProcess_Pipe_STDERR) {
-      if (!err->Process(data, length)) {
+      processOutput.DecodeText(data, length, strdata, 2);
+      if (!err->Process(strdata.c_str(), int(strdata.size()))) {
         err = CM_NULLPTR;
       }
     }
   }
+  if (out) {
+    processOutput.DecodeText(std::string(), strdata, 1);
+    if (!strdata.empty()) {
+      out->Process(strdata.c_str(), int(strdata.size()));
+    }
+  }
+  if (err) {
+    processOutput.DecodeText(std::string(), strdata, 2);
+    if (!strdata.empty()) {
+      out->Process(strdata.c_str(), int(strdata.size()));
+    }
+  }
   cmsysProcess_WaitForExit(cp, CM_NULLPTR);
 }
 

Source/cmSystemTools.cxx

@@ -3,6 +3,7 @@
 #include "cmSystemTools.h"
 
 #include "cmAlgorithms.h"
+#include "cmProcessOutput.h"
 
 #if defined(CMAKE_BUILD_WITH_CMAKE)
 #include "cmArchiveWrite.h"
@@ -609,6 +610,8 @@ bool cmSystemTools::RunSingleCommand(std::vector<std::string> const& command,
   char* data;
   int length;
   int pipe;
+  cmProcessOutput processOutput;
+  std::string strdata;
   if (outputflag != OUTPUT_PASSTHROUGH &&
       (captureStdOut || captureStdErr || outputflag != OUTPUT_NONE)) {
     while ((pipe = cmsysProcess_WaitForData(cp, &data, &length, CM_NULLPTR)) >
@@ -624,28 +627,44 @@ bool cmSystemTools::RunSingleCommand(std::vector<std::string> const& command,
 
       if (pipe == cmsysProcess_Pipe_STDOUT) {
         if (outputflag != OUTPUT_NONE) {
-          cmSystemTools::Stdout(data, length);
+          processOutput.DecodeText(data, length, strdata, 1);
+          cmSystemTools::Stdout(strdata.c_str(), strdata.size());
         }
         if (captureStdOut) {
           tempStdOut.insert(tempStdOut.end(), data, data + length);
         }
       } else if (pipe == cmsysProcess_Pipe_STDERR) {
         if (outputflag != OUTPUT_NONE) {
-          cmSystemTools::Stderr(data, length);
+          processOutput.DecodeText(data, length, strdata, 2);
+          cmSystemTools::Stderr(strdata.c_str(), strdata.size());
         }
         if (captureStdErr) {
           tempStdErr.insert(tempStdErr.end(), data, data + length);
         }
       }
     }
+
+    if (outputflag != OUTPUT_NONE) {
+      processOutput.DecodeText(std::string(), strdata, 1);
+      if (!strdata.empty()) {
+        cmSystemTools::Stdout(strdata.c_str(), strdata.size());
+      }
+      processOutput.DecodeText(std::string(), strdata, 2);
+      if (!strdata.empty()) {
+        cmSystemTools::Stderr(strdata.c_str(), strdata.size());
+      }
+    }
   }
 
   cmsysProcess_WaitForExit(cp, CM_NULLPTR);
+
   if (captureStdOut) {
     captureStdOut->assign(tempStdOut.begin(), tempStdOut.end());
+    processOutput.DecodeText(*captureStdOut, *captureStdOut);
   }
   if (captureStdErr) {
     captureStdErr->assign(tempStdErr.begin(), tempStdErr.end());
+    processOutput.DecodeText(*captureStdErr, *captureStdErr);
   }
 
   bool result = true;
@@ -1643,6 +1662,8 @@ int cmSystemTools::WaitForLine(cmsysProcess* process, std::string& line,
   line = "";
   std::vector<char>::iterator outiter = out.begin();
   std::vector<char>::iterator erriter = err.begin();
+  cmProcessOutput processOutput;
+  std::string strdata;
   while (1) {
     // Check for a newline in stdout.
     for (; outiter != out.end(); ++outiter) {
@@ -1687,17 +1708,31 @@ int cmSystemTools::WaitForLine(cmsysProcess* process, std::string& line,
       return pipe;
     }
     if (pipe == cmsysProcess_Pipe_STDOUT) {
+      processOutput.DecodeText(data, length, strdata, 1);
       // Append to the stdout buffer.
       std::vector<char>::size_type size = out.size();
-      out.insert(out.end(), data, data + length);
+      out.insert(out.end(), strdata.begin(), strdata.end());
       outiter = out.begin() + size;
     } else if (pipe == cmsysProcess_Pipe_STDERR) {
+      processOutput.DecodeText(data, length, strdata, 2);
       // Append to the stderr buffer.
       std::vector<char>::size_type size = err.size();
-      err.insert(err.end(), data, data + length);
+      err.insert(err.end(), strdata.begin(), strdata.end());
       erriter = err.begin() + size;
     } else if (pipe == cmsysProcess_Pipe_None) {
       // Both stdout and stderr pipes have broken.  Return leftover data.
+      processOutput.DecodeText(std::string(), strdata, 1);
+      if (!strdata.empty()) {
+        std::vector<char>::size_type size = out.size();
+        out.insert(out.end(), strdata.begin(), strdata.end());
+        outiter = out.begin() + size;
+      }
+      processOutput.DecodeText(std::string(), strdata, 2);
+      if (!strdata.empty()) {
+        std::vector<char>::size_type size = err.size();
+        err.insert(err.end(), strdata.begin(), strdata.end());
+        erriter = err.begin() + size;
+      }
       if (!out.empty()) {
         line.append(&out[0], outiter - out.begin());
         out.erase(out.begin(), out.end());