cmake: make tests successfully passing part of the

Question

I am trying to make the passing of tests part of the build process.

Here I use add_custom_command to run the test as a POST_BUILD step.

function(register_test NAME)

    add_test(${NAME} ${NAME})

    # make the test run as part of the build process
    add_custom_command(TARGET ${NAME} POST_BUILD COMMAND ${NAME})

endfunction()

The problem with this approach is that the test is only run when the target is built:

$ make

[ 50%] Built target lib1
Linking CXX executable ../../Debug/bin/lib1_test
Running 1 test case...
main.cpp(8): fatal error: in "lib1_test": 
    critical check lib1() == "lib1" has failed [error != lib1]

*** 1 failure is detected in the test module "Master Test Suite"

make[2]: *** [lib1/test/lib1_test] Error 201
make[1]: *** [lib1/test/CMakeFiles/lib1_test.dir/all] Error 2
make: *** [all] Error 2

If the target doesn't need to be built, then the test is not run, and the build passes.

Here I don't make any changes, just rerun the build process

$ make

[ 50%] Built target lib1
[100%] Built target lib1_test

However, if lib1_test is actually run, the test fails.

$ ./lib1/test/lib1_test 

Running 1 test case...
main.cpp(8): fatal error: in "lib1_test": 
    critical check lib1() == "lib1" has failed [error != lib1]

*** 1 failure is detected in the test module "Master Test Suite"

A better way to do this would be to make a lib1_test.passed target which depends on lib1_test, runs the tests, and is only created if the tests pass.

What I have tried:

I have tried using add_custom_target to create a target lib1_test.passed which depends on lib1_test, and if successful, creates a file lib1_test.passed:

add_custom_target(${NAME}.passed
    DEPENDS ${NAME}
    COMMAND ${NAME}
    COMMAND ${CMAKE_COMMAND} -E touch ${NAME}.passed)

There are 2 shortfalls with what I have currently achieved:

• The running of the test is not part of the normal build process.
  That is, make will not "build" lib1_test.passed;
  I have to explicitly state make lib1_test.passed
• make lib1_test.passed will always execute lib1_test, regardless of whether lib1_test.passed is newer than lib1_test1 or not

Question:

How can I make the running of tests part of the build, where a failing test will be always rerun?

Answer 1

As shown in @user3159253 answer you need an output file (with a time stamp) for the build environment to check if it has to "build" the target in question again. So if your executable was successfully build - even if the subsequent call to run it fails - won't build again.

I wanted to add a solution where you can have only one target. It will rename the test executable output, run it and - if successful - rename it again to its original name to sort of mark it as "passed":

function(register_test NAME)

    add_test(NAME ${NAME} COMMAND ${NAME})

    set(TMP "$<TARGET_FILE_DIR:${NAME}>/tmp_$<TARGET_FILE_NAME:${NAME}>")
    set(ORG "$<TARGET_FILE:${NAME}>")

    # make the test run as part of the build process
    add_custom_command(
        TARGET ${NAME} 
        POST_BUILD 
        COMMAND ${CMAKE_COMMAND} -E rename "${ORG}" "${TMP}"
        COMMAND ${TMP}
        COMMAND ${CMAKE_COMMAND} -E rename "${TMP}" "${ORG}"
    )

endfunction()

Answer 2

Here what I've got so far. The implementation is pretty quick and dirty but nevertheless it works. Please check and tell if it satisfies your needs.

CMakeLists.txt:

cmake_minimum_required(VERSION 2.8.12)

project(test)

enable_testing()

set(lib1_SRC lib.c)

add_library(lib1 ${lib1_SRC})

set(test_SRC test.c)

add_executable(libtest ${test_SRC})
target_link_libraries(libtest lib1)

add_test(NAME libtest COMMAND libtest)

add_custom_command(
  OUTPUT _libtest_completed
  COMMAND ctest -C $<CONFIGURATION> --output-on-failure
  COMMAND cmake -E touch _libtest_completed
  DEPENDS libtest
)

add_custom_target(
  libtest_force ALL
  DEPENDS _libtest_completed
)

Source files for the sake of completeness:

lib.c:

#include "lib.h"

#include <time.h>

int lib_func() {
    return time(NULL) % 2;
}

lib.h:

#pragma once

int lib_func();

test.c:

#include "lib.h"

int main() {
    return lib_func();
}

Unfortunately it's impossible to depend directly on test target due to CMake bug so we have to perform sunset manually.