CI Changes

test/NewTestSuite.py

@@ -10,6 +10,16 @@ from collections import OrderedDict
 cpu_threads = multiprocessing.cpu_count()
 py_version = f"{sys.version_info.major}.{sys.version_info.minor}"
 
+PYLINE = "python"
+PIPLINE_PATH = os.path.join(".","resources","user","meta","manifests","pipline.txt")
+if os.path.isfile(PIPLINE_PATH):
+    with open(PIPLINE_PATH) as pipline_file:
+        PYLINE = pipline_file.read().replace("-m pip","").strip()
+
+results = {
+    "errors": [],
+    "success": []
+}
 
 def main(args=None):
     successes = []
@@ -28,7 +38,7 @@ def main(args=None):
     def test(test_name: str, command: str, test_file: str):
         tests[test_name] = [command]
 
-        base_command = f"python3 DungeonRandomizer.py --suppress_rom --suppress_spoiler"
+        base_command = f"{PYLINE} DungeonRandomizer.py --suppress_rom --jsonout --spoiler none"
 
         def gen_seed():
             task_command = base_command + " " + command
@@ -102,7 +112,7 @@ if __name__ == "__main__":
 
     test_suites = {}
     # not sure if it supports subdirectories properly yet
-    for root, dirnames, filenames in os.walk('test/suite'):
+    for root, dirnames, filenames in os.walk(os.path.join("test","suite")):
         test_suites[root] = fnmatch.filter(filenames, '*.yaml')
 
     args = argparse.Namespace()
@@ -113,14 +123,30 @@ if __name__ == "__main__":
     successes += s
     print()
 
+    LOGPATH = os.path.join(".","logs")
+    if not os.path.isdir(LOGPATH):
+        os.makedirs(LOGPATH)
+
     if errors:
-        with open(f"new-test-suite-errors.txt", 'w') as stream:
+        with open(os.path.join(LOGPATH, "new-test-suite-errors.txt"), 'w') as stream:
             for error in errors:
                 stream.write(error[0] + "\n")
                 stream.write(error[1] + "\n")
                 stream.write(error[2] + "\n\n")
+                error[2] = error[2].split("\n")
+                results["errors"].append(error)
 
-    with open("new-test-suite-success.txt", "w") as stream:
+    with open(os.path.join(LOGPATH, "new-test-suite-success.txt"), 'w') as stream:
         stream.write(str.join("\n", successes))
+        results["success"] = successes
 
-    input("Press enter to continue")
+    num_errors  = len(results["errors"])
+    num_success = len(results["success"])
+    num_total   = num_errors + num_success
+
+    print(f"Errors:  {num_errors}/{num_total}")
+    print(f"Success: {num_success}/{num_total}")
+    # print(results)
+
+    if (num_errors/num_total) > (num_success/num_total):
+        exit(1)