From 6305b40a331e877c8fd608ebe5fe40088190f204 Mon Sep 17 00:00:00 2001
From: Ryan Houdek <Sonicadvance1@gmail.com>
Date: Mon, 16 Mar 2020 12:35:09 -0700
Subject: [PATCH] Adds known failure support to unit test runner

This will allow us to add unit tests that have known failures like
unsupported features

Additionally if the test passes and is on the known failure list then it
still fails since its state has changed
---
 Scripts/testharness_runner.py | 41 +++++++++++++++++++++++++++++++++++
 unittests/ASM/CMakeLists.txt  |  6 ++++-
 unittests/ASM/Known_Failures  |  4 ++++
 3 files changed, 50 insertions(+), 1 deletion(-)
 create mode 100644 Scripts/testharness_runner.py
 create mode 100644 unittests/ASM/Known_Failures

diff --git a/Scripts/testharness_runner.py b/Scripts/testharness_runner.py
new file mode 100644
index 000000000..6c977405d
--- /dev/null
+++ b/Scripts/testharness_runner.py
@@ -0,0 +1,41 @@
+#!/usr/bin/python3
+import sys
+import subprocess
+
+# Args: <Known Failures file> <TestName> <Test Harness Executable> <Args>...
+
+if (len(sys.argv) < 4):
+    sys.exit()
+
+known_failures = {}
+known_failures_file = sys.argv[1]
+
+current_test = sys.argv[2]
+
+# Open the known failures file and add it to a dictionary
+with open(known_failures_file) as kff:
+    for line in kff:
+        known_failures[line.strip()] = 1
+
+RunnerArgs = ["catchsegv", sys.argv[3]]
+# Add the rest of the arguments
+for i in range(len(sys.argv) - 4):
+    RunnerArgs.append(sys.argv[4 + i])
+
+# Run the test and wait for it to end to get the result
+Process = subprocess.Popen(RunnerArgs)
+Process.wait()
+ResultCode = Process.returncode
+
+if (known_failures.get(current_test)):
+    # If the test is on the known failures list
+    if (ResultCode):
+        # If we errored but are on the known failures list then "pass" the test
+        sys.exit(0)
+    else:
+        # If we didn't error but are in the known failure list then we need to fail the test
+        sys.exit(1)
+else:
+    # Just return the result code if we don't have this test as a known failure
+    sys.exit(ResultCode);
+
diff --git a/unittests/ASM/CMakeLists.txt b/unittests/ASM/CMakeLists.txt
index fbc897afd..ffa12671c 100644
--- a/unittests/ASM/CMakeLists.txt
+++ b/unittests/ASM/CMakeLists.txt
@@ -45,7 +45,11 @@ foreach(ASM_SRC ${ASM_SOURCES})
     set(TEST_NAME "${TEST_NUM}/Test_${ASM_NAME}")
     string(REPLACE " " ";" ARGS_LIST ${ARGS})
     add_test(NAME ${TEST_NAME}
-      COMMAND "${CMAKE_BINARY_DIR}/Bin/TestHarnessRunner" ${ARGS_LIST} "${OUTPUT_NAME}" "${OUTPUT_CONFIG_NAME}")
+      COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/testharness_runner.py"
+      "${CMAKE_SOURCE_DIR}/unittests/ASM/Known_Failures"
+      "Test_${ASM_NAME}"
+      "${CMAKE_BINARY_DIR}/Bin/TestHarnessRunner"
+      ${ARGS_LIST} "${OUTPUT_NAME}" "${OUTPUT_CONFIG_NAME}")
     # This will cause the ASM tests to fail if it can't find the TestHarness or ASMN files
     # Prety crap way to work around the fact that tests can't have a build dependency in a different directory
     # Just make sure to independently run `make all` then `make test`
diff --git a/unittests/ASM/Known_Failures b/unittests/ASM/Known_Failures
new file mode 100644
index 000000000..5ef292b3a
--- /dev/null
+++ b/unittests/ASM/Known_Failures
@@ -0,0 +1,4 @@
+Test_14_XX_02.asm
+Test_0F_2E.asm
+Test_0F_2F.asm
+Test_fld.asm