diff --git a/Scripts/testharness_runner.py b/Scripts/testharness_runner.py new file mode 100644 index 000000000..6c977405d --- /dev/null +++ b/Scripts/testharness_runner.py @@ -0,0 +1,41 @@ +#!/usr/bin/python3 +import sys +import subprocess + +# Args: ... + +if (len(sys.argv) < 4): + sys.exit() + +known_failures = {} +known_failures_file = sys.argv[1] + +current_test = sys.argv[2] + +# Open the known failures file and add it to a dictionary +with open(known_failures_file) as kff: + for line in kff: + known_failures[line.strip()] = 1 + +RunnerArgs = ["catchsegv", sys.argv[3]] +# Add the rest of the arguments +for i in range(len(sys.argv) - 4): + RunnerArgs.append(sys.argv[4 + i]) + +# Run the test and wait for it to end to get the result +Process = subprocess.Popen(RunnerArgs) +Process.wait() +ResultCode = Process.returncode + +if (known_failures.get(current_test)): + # If the test is on the known failures list + if (ResultCode): + # If we errored but are on the known failures list then "pass" the test + sys.exit(0) + else: + # If we didn't error but are in the known failure list then we need to fail the test + sys.exit(1) +else: + # Just return the result code if we don't have this test as a known failure + sys.exit(ResultCode); + diff --git a/unittests/ASM/CMakeLists.txt b/unittests/ASM/CMakeLists.txt index fbc897afd..ffa12671c 100644 --- a/unittests/ASM/CMakeLists.txt +++ b/unittests/ASM/CMakeLists.txt @@ -45,7 +45,11 @@ foreach(ASM_SRC ${ASM_SOURCES}) set(TEST_NAME "${TEST_NUM}/Test_${ASM_NAME}") string(REPLACE " " ";" ARGS_LIST ${ARGS}) add_test(NAME ${TEST_NAME} - COMMAND "${CMAKE_BINARY_DIR}/Bin/TestHarnessRunner" ${ARGS_LIST} "${OUTPUT_NAME}" "${OUTPUT_CONFIG_NAME}") + COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/testharness_runner.py" + "${CMAKE_SOURCE_DIR}/unittests/ASM/Known_Failures" + "Test_${ASM_NAME}" + "${CMAKE_BINARY_DIR}/Bin/TestHarnessRunner" + ${ARGS_LIST} "${OUTPUT_NAME}" "${OUTPUT_CONFIG_NAME}") # This will cause the ASM tests to fail if it can't find the TestHarness or ASMN files # Prety crap way to work around the fact that tests can't have a build dependency in a different directory # Just make sure to independently run `make all` then `make test` diff --git a/unittests/ASM/Known_Failures b/unittests/ASM/Known_Failures new file mode 100644 index 000000000..5ef292b3a --- /dev/null +++ b/unittests/ASM/Known_Failures @@ -0,0 +1,4 @@ +Test_14_XX_02.asm +Test_0F_2E.asm +Test_0F_2F.asm +Test_fld.asm