QA: Add build-time option to enable XML output from tests

This allows CI tools to get machine-readable test reports.

By default, this is off - when you test normally, you probably
want the regular console spew.

Boost tests use the in-built Boost test command line options.

The Python tests take an --xml parameter and output the tests
there.

CMakeModules/KiCadQABuildUtils.cmake

@@ -24,6 +24,14 @@
 # tests and tools
 #
 
+# CI tools can use this to force tests to output machine-readable formats
+# when possible
+option( KICAD_TEST_XML_OUTPUT
+    "Cause unit tests to output xUnit results where possible for more granular CI reporting."
+    OFF
+)
+mark_as_advanced( KICAD_TEST_XML_OUTPUT ) # Only CI tools need this
+
 # This is a "meta" target that is used to collect all tests
 add_custom_target( qa_all_tests )
 
@@ -41,9 +49,19 @@ add_custom_target( qa_all
 #   * Is a dependency of qa_all_tests
 function( kicad_add_boost_test TEST_EXEC_TARGET TEST_NAME)
 
+set(BOOST_TEST_PARAMS "")
+
+if( KICAD_TEST_XML_OUTPUT )
+    # Provide Boost-test-y XML params if asked
+    # Due to Boost issue in 1.62, have to use the --logger parameter, rather than
+    # separate --log_format, --log_sink, etc parameter
+    # https://svn.boost.org/trac10/ticket/12507
+    set(BOOST_TEST_PARAMS --logger=XML,all,${TEST_NAME}.boost-results.xml --report_level=no --result_code=no)
+endif()
+
 # Add the test to the CTest registry
 add_test( NAME ${TEST_NAME}
-    COMMAND $<TARGET_FILE:${TEST_EXEC_TARGET}>
+    COMMAND $<TARGET_FILE:${TEST_EXEC_TARGET}> ${BOOST_TEST_PARAMS}
 )
 
 # Make the overall test meta-target depend on this test

Documentation/development/testing.md

@@ -44,6 +44,22 @@ Common useful patterns:
 You can rebuild just a specific test with CMake to avoid rebuilding
 everything when working on a small area, e.g. `make qa_common`.
 
+### Automated testing {#automated-testing}
+
+The unit tests can be run on automated Continuous Integration (CI) systems.
+
+By default, tests output human-readable results, which is useful when
+developing or debugging, but not so useful for automated test reporting.
+Systems that can parse XML test results can enable these by setting the
+`KICAD_TEST_XML_OUTPUT` option to `ON`. The test results are then output
+as files ending in `.xml` in the `qa` subdirectory.
+
+Test results are written to the build directory as follows:
+
+* Boost units tests: one XML file per test with the extension `.boost-results.xml`
+* Python unit tests: one directory per test with the extension `.xunit-results.xml`.
+  These directories contain one `.xml` file per Python test case file.
+
 ## Writing Boost tests {#writing-boost-tests}
 
 Boost unit tests are straightforward to write. Individual test cases can be

qa/CMakeLists.txt

@@ -23,12 +23,18 @@ include(KiCadQABuildUtils)
 
 if( KICAD_SCRIPTING_MODULES )
 
+    if( KICAD_TEST_XML_OUTPUT )
+        # To do this, you will need xmlrunner
+        set( PY_TEST_ARGS --xml=${CMAKE_CURRENT_BINARY_DIR}/python.xunit-results )
+    endif()
+
     # Test that runs the QA tests through scripting
     add_test( NAME qa_python
-        COMMAND ${PYTHON_EXECUTABLE} test.py
+        COMMAND ${PYTHON_EXECUTABLE} test.py ${PY_TEST_ARGS}
         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
     )
 
+
     set_property( TEST qa_python
         PROPERTY ENVIRONMENT "PYTHONPATH=${CMAKE_BINARY_DIR}/pcbnew${PYTHON_QA_PATH}"
     )

qa/test.py

@@ -1,18 +1,45 @@
 import unittest
 import platform
 import sys
+import argparse
 
 if platform.python_version() < '2.7':
     unittest = __import__('unittest2')
 else:
     import unittest
 
+try:
+    import xmlrunner
+    have_xml = True
+except ImportError:
+    have_xml = False
+
+
 if __name__ == '__main__':
-    testsuite = unittest.TestLoader().discover('testcases',pattern="*.py")
-    results = unittest.TextTestRunner(verbosity=100).run(testsuite)
+
+    parser = argparse.ArgumentParser(
+            description='Test suit for KiCad Python functions')
+    parser.add_argument('--xml', action="store", type=str,
+                        help='Output XML test results to the given directory')
+
+    args = parser.parse_args()
+
+    if args.xml and not have_xml:
+        print("XML test reporting not available")
+        print("Install the xmlrunner package.")
+        sys.exit(2)
+
+    testsuite = unittest.TestLoader().discover('testcases', pattern="*.py")
+
+    if args.xml:
+        # Dump XML results to the right directory
+        runner = xmlrunner.XMLTestRunner(output=args.xml)
+    else:
+        # Use a normal text runner
+        runner = unittest.TextTestRunner(verbosity=100)
+
+    results = runner.run(testsuite)
 
     # Return an error code if any of the testsuite tests fail
     if not results.wasSuccessful():
         sys.exit(1)
-
-