Merge pull request #34 from Sonicadvance1/unittests/known_failures

Adds known failure support to unit test runner

Scripts/testharness_runner.py

@@ -0,0 +1,41 @@
+#!/usr/bin/python3
+import sys
+import subprocess
+
+# Args: <Known Failures file> <TestName> <Test Harness Executable> <Args>...
+
+if (len(sys.argv) < 4):
+    sys.exit()
+
+known_failures = {}
+known_failures_file = sys.argv[1]
+
+current_test = sys.argv[2]
+
+# Open the known failures file and add it to a dictionary
+with open(known_failures_file) as kff:
+    for line in kff:
+        known_failures[line.strip()] = 1
+
+RunnerArgs = ["catchsegv", sys.argv[3]]
+# Add the rest of the arguments
+for i in range(len(sys.argv) - 4):
+    RunnerArgs.append(sys.argv[4 + i])
+
+# Run the test and wait for it to end to get the result
+Process = subprocess.Popen(RunnerArgs)
+Process.wait()
+ResultCode = Process.returncode
+
+if (known_failures.get(current_test)):
+    # If the test is on the known failures list
+    if (ResultCode):
+        # If we errored but are on the known failures list then "pass" the test
+        sys.exit(0)
+    else:
+        # If we didn't error but are in the known failure list then we need to fail the test
+        sys.exit(1)
+else:
+    # Just return the result code if we don't have this test as a known failure
+    sys.exit(ResultCode);
+

unittests/ASM/CMakeLists.txt

@@ -45,7 +45,11 @@ foreach(ASM_SRC ${ASM_SOURCES})
     set(TEST_NAME "${TEST_NUM}/Test_${ASM_NAME}")
     string(REPLACE " " ";" ARGS_LIST ${ARGS})
     add_test(NAME ${TEST_NAME}
-      COMMAND "${CMAKE_BINARY_DIR}/Bin/TestHarnessRunner" ${ARGS_LIST} "${OUTPUT_NAME}" "${OUTPUT_CONFIG_NAME}")
+      COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/testharness_runner.py"
+      "${CMAKE_SOURCE_DIR}/unittests/ASM/Known_Failures"
+      "Test_${ASM_NAME}"
+      "${CMAKE_BINARY_DIR}/Bin/TestHarnessRunner"
+      ${ARGS_LIST} "${OUTPUT_NAME}" "${OUTPUT_CONFIG_NAME}")
     # This will cause the ASM tests to fail if it can't find the TestHarness or ASMN files
     # Prety crap way to work around the fact that tests can't have a build dependency in a different directory
     # Just make sure to independently run `make all` then `make test`

unittests/ASM/Known_Failures

@@ -0,0 +1,4 @@
+Test_14_XX_02.asm
+Test_0F_2E.asm
+Test_0F_2F.asm
+Test_fld.asm