ctest: Add JSON schema for --show-only=json-v1 output

Add schema validation to the existing test case for --show-only=json-v1 too.

Fixes: #26980

Help/manual/ctest.1.rst

@@ -1622,9 +1622,11 @@ model is defined as follows:
   A JSON object specifying the version components.  Its members are:
 
   ``major``
-    A non-negative integer specifying the major version component.
+    A positive integer specifying the major version component
+    of the JSON object model.
   ``minor``
-    A non-negative integer specifying the minor version component.
+    A non-negative integer specifying the minor version component
+    of the JSON object model.
 
 ``backtraceGraph``
     JSON object representing backtrace information with the
@@ -1683,6 +1685,10 @@ model is defined as follows:
       The property value, which can be a string, a number, a boolean, or an
       array of strings.
 
+.. versionadded:: 4.1
+  The JSON output format is described in machine-readable form by
+  :download:`this JSON schema </manual/ctest/show-only-schema.json>`.
+
 .. _`ctest-resource-allocation`:
 
 Resource Allocation

Help/manual/ctest/show-only-schema.json

@@ -0,0 +1,121 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "required": ["kind", "version", "backtraceGraph", "tests"],
+  "properties": {
+    "kind": {
+      "type": "string",
+      "const": "ctestInfo"
+    },
+    "version": {
+      "type": "object",
+      "required": ["major", "minor"],
+      "properties": {
+        "major": {
+          "const": 1,
+          "description": "A positive integer specifying the major version component of the JSON object model."
+        },
+        "minor": {
+          "const": 0,
+          "description": "A non-negative integer specifying the minor version component of the JSON object model."
+        }
+      }
+    },
+    "backtraceGraph": {
+      "type": "object",
+      "required": ["commands", "files", "nodes"],
+      "properties": {
+        "commands": {
+          "type": "array",
+          "description": "List of CMake command names.",
+          "items": {
+            "type": "string"
+          }
+        },
+        "files": {
+          "type": "array",
+          "description": "List of file paths, which may be relative or absolute. Relative paths are relative to the top-level source directory.",
+          "items": {
+            "type": "string"
+          }
+        },
+        "nodes": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["file"],
+            "properties": {
+              "command": {
+                "type": "integer",
+                "minimum": 0,
+                "description": "An optional member present when the node represents a command invocation within the file. The value is an unsigned integer 0-based index into the commands member of the backtraceGraph."
+              },
+              "file": {
+                "type": "integer",
+                "minimum": 0,
+                "description": "An unsigned integer 0-based index into the files member of the backtraceGraph."
+              },
+              "line": {
+                "type": "integer",
+                "minimum": 1,
+                "description": "An optional member present when the node represents a line within the file. The value is an unsigned integer 1-based line number in the file where the backtrace was added."
+              },
+              "parent": {
+                "type": "integer",
+                "minimum": 0,
+                "description": "An optional member present when the node is not the bottom of the call stack. The value is an unsigned integer 0-based index into the nodes member of the backtraceGraph representing the parent in the graph."
+              }
+            }
+          }
+        }
+      }
+    },
+    "tests": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["name", "backtrace"],
+        "properties": {
+          "name": {
+            "type": "string",
+            "description": "Test name",
+            "minLength": 1
+          },
+          "config": {
+            "type": "string",
+            "description": "Optional field specifying the configuration for which the test will run. This will always match the -C option specified on the ctest command line. If no such option was given, this field will not be present."
+          },
+          "command": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "minItems": 1,
+            "description": "Optional array where the first element is the test command and the remaining elements are the command arguments. Normally, this field should be present and non-empty, but in certain corner cases involving generator expressions, it is possible for a test to have no command and therefore this field can be missing."
+          },
+          "backtrace": {
+            "type": "integer",
+            "description": "Index into the nodes member of the backtraceGraph."
+          },
+          "properties": {
+            "type": "array",
+            "description": "Optional list of test properties associated with the test.",
+            "items": {
+              "type": "object",
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "Test property name.",
+                  "minLength": 1
+                },
+                "value": {
+                  "description": "Value of the test property. Any valid JSON type might be present."
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

Tests/RunCMake/CMakeLists.txt

@@ -1084,7 +1084,10 @@ if(CMake_TEST_RunCMake_ExternalProject_RUN_SERIAL)
 endif()
 add_RunCMake_test(FetchContent)
 add_RunCMake_test(FetchContent_find_package)
-set(CTestCommandLine_ARGS -DPython_EXECUTABLE=${Python_EXECUTABLE})
+set(CTestCommandLine_ARGS
+  -DPython_EXECUTABLE=${Python_EXECUTABLE}
+  -DCMake_TEST_JSON_SCHEMA=${CMake_TEST_JSON_SCHEMA}
+)
 if(NOT CMake_TEST_EXTERNAL_CMAKE)
   list(APPEND CTestCommandLine_ARGS -DTEST_AFFINITY=$<TARGET_FILE:testAffinity>)
 endif()

Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake

@@ -1,5 +1,6 @@
 include(RunCMake)
 include(RunCTest)
+cmake_policy(SET CMP0140 NEW)
 
 # Do not use any proxy for lookup of an invalid site.
 # DNS failure by proxy looks different than DNS failure without proxy.
@@ -449,6 +450,20 @@ function(show_only_json_check_python v)
   set(json_file "${RunCMake_TEST_BINARY_DIR}/ctest.json")
   file(WRITE "${json_file}" "${actual_stdout}")
   set(actual_stdout "" PARENT_SCOPE)
+
+  if(CMake_TEST_JSON_SCHEMA)
+    execute_process(
+      COMMAND ${Python_EXECUTABLE} "${RunCMake_SOURCE_DIR}/show-only_json_validate_schema.py" "${json_file}"
+      RESULT_VARIABLE result
+      OUTPUT_VARIABLE output
+      ERROR_VARIABLE output
+    )
+    if(NOT result STREQUAL 0)
+      string(REPLACE "\n" "\n  " output "${output}")
+      string(APPEND RunCMake_TEST_FAILED "Failed to validate version ${v} JSON schema for file: ${file}\nOutput:\n${output}\n")
+    endif()
+  endif()
+
   execute_process(
     COMMAND ${Python_EXECUTABLE} "${RunCMake_SOURCE_DIR}/show-only_json-v${v}_check.py" "${json_file}"
     RESULT_VARIABLE result
@@ -457,8 +472,9 @@ function(show_only_json_check_python v)
     )
   if(NOT result EQUAL 0)
     string(REPLACE "\n" "\n  " output "  ${output}")
-    set(RunCMake_TEST_FAILED "Unexpected output:\n${output}" PARENT_SCOPE)
+    string(APPEND RunCMake_TEST_FAILED "Unexpected output:\n${output}" PARENT_SCOPE)
   endif()
+  return(PROPAGATE RunCMake_TEST_FAILED)
 endfunction()
 
 function(run_ShowOnly)

Tests/RunCMake/CTestCommandLine/show-only_json_validate_schema.py

@@ -0,0 +1,16 @@
+import json
+import jsonschema
+import os.path
+import sys
+
+
+with open(sys.argv[1], "r", encoding="utf-8-sig") as f:
+    contents = json.load(f)
+
+schema_file = os.path.join(
+    os.path.dirname(__file__),
+    "..", "..", "..", "Help", "manual", "ctest", "show-only-schema.json")
+with open(schema_file, "r", encoding="utf-8") as f:
+    schema = json.load(f)
+
+jsonschema.validate(contents, schema)