CI Changes

test/MysteryTestSuite.py

@@ -1,3 +1,4 @@
+import os
 import subprocess
 import sys
 import multiprocessing
@@ -8,6 +9,16 @@ from collections import OrderedDict
 cpu_threads = multiprocessing.cpu_count()
 py_version = f"{sys.version_info.major}.{sys.version_info.minor}"
 
+PYLINE = "python"
+PIPLINE_PATH = os.path.join(".","resources","user","meta","manifests","pipline.txt")
+if os.path.isfile(PIPLINE_PATH):
+    with open(PIPLINE_PATH) as pipline_file:
+        PYLINE = pipline_file.read().replace("-m pip","").strip()
+
+results = {
+    "errors": [],
+    "success": []
+}
 
 def main(args=None):
     successes = []
@@ -25,7 +36,7 @@ def main(args=None):
 
     def test(testname: str, command: str):
         tests[testname] = [command]
-        basecommand = f"python3.8 Mystery.py --suppress_rom --suppress_meta"
+        basecommand = f"{PYLINE} Mystery.py --suppress_rom --suppress_meta"
 
         def gen_seed():
             taskcommand = basecommand + " " + command
@@ -98,6 +109,10 @@ if __name__ == "__main__":
 
     cpu_threads = args.cpu_threads
 
+    LOGPATH = os.path.join(".","logs")
+    if not os.path.isdir(LOGPATH):
+        os.makedirs(LOGPATH)
+
     for dr in [['mystery', args.count if args.count else 1, 1]]:
 
         for tense in range(1, dr[2] + 1):
@@ -112,13 +127,36 @@ if __name__ == "__main__":
             print()
 
             if errors:
-                with open(f"{dr[0]}{(f'-{tense}' if dr[0] in ['basic', 'crossed'] else '')}-errors.txt", 'w') as stream:
+                errors_filename = f"{dr[0]}"
+                if dr[0] in ["basic","crossed"]:
+                    errors_filename += f"-{tense}"
+                errors_filename += "-errors.txt"
+                with open(
+                    os.path.join(
+                        LOGPATH,
+                        errors_filename
+                    ),
+                    'w'
+                ) as stream:
                     for error in errors:
                         stream.write(error[0] + "\n")
                         stream.write(error[1] + "\n")
                         stream.write(error[2] + "\n\n")
+                        error[2] = error[2].split("\n")
+                        results["errors"].append(error)
 
-    with open("success.txt", "w") as stream:
+    with open(os.path.join(LOGPATH, "mystery-success.txt"), "w") as stream:
         stream.write(str.join("\n", successes))
+        results["success"] = successes
 
-    input("Press enter to continue")
+    num_errors  = len(results["errors"])
+    num_success = len(results["success"])
+    num_total   = num_errors + num_success
+
+    print(f"Errors:  {num_errors}/{num_total}")
+    print(f"Success: {num_success}/{num_total}")
+    # print(results)
+
+    if (num_errors/num_total) > (num_success/num_total):
+        # exit(1)
+        pass

test/NewTestSuite.py

@@ -10,6 +10,16 @@ from collections import OrderedDict
 cpu_threads = multiprocessing.cpu_count()
 py_version = f"{sys.version_info.major}.{sys.version_info.minor}"
 
+PYLINE = "python"
+PIPLINE_PATH = os.path.join(".","resources","user","meta","manifests","pipline.txt")
+if os.path.isfile(PIPLINE_PATH):
+    with open(PIPLINE_PATH) as pipline_file:
+        PYLINE = pipline_file.read().replace("-m pip","").strip()
+
+results = {
+    "errors": [],
+    "success": []
+}
 
 def main(args=None):
     successes = []
@@ -28,7 +38,7 @@ def main(args=None):
     def test(test_name: str, command: str, test_file: str):
         tests[test_name] = [command]
 
-        base_command = f"python3 DungeonRandomizer.py --suppress_rom --suppress_spoiler"
+        base_command = f"{PYLINE} DungeonRandomizer.py --suppress_rom --jsonout --spoiler none"
 
         def gen_seed():
             task_command = base_command + " " + command
@@ -102,7 +112,7 @@ if __name__ == "__main__":
 
     test_suites = {}
     # not sure if it supports subdirectories properly yet
-    for root, dirnames, filenames in os.walk('test/suite'):
+    for root, dirnames, filenames in os.walk(os.path.join("test","suite")):
         test_suites[root] = fnmatch.filter(filenames, '*.yaml')
 
     args = argparse.Namespace()
@@ -113,14 +123,30 @@ if __name__ == "__main__":
     successes += s
     print()
 
+    LOGPATH = os.path.join(".","logs")
+    if not os.path.isdir(LOGPATH):
+        os.makedirs(LOGPATH)
+
     if errors:
-        with open(f"new-test-suite-errors.txt", 'w') as stream:
+        with open(os.path.join(LOGPATH, "new-test-suite-errors.txt"), 'w') as stream:
             for error in errors:
                 stream.write(error[0] + "\n")
                 stream.write(error[1] + "\n")
                 stream.write(error[2] + "\n\n")
+                error[2] = error[2].split("\n")
+                results["errors"].append(error)
 
-    with open("new-test-suite-success.txt", "w") as stream:
+    with open(os.path.join(LOGPATH, "new-test-suite-success.txt"), 'w') as stream:
         stream.write(str.join("\n", successes))
+        results["success"] = successes
 
-    input("Press enter to continue")
+    num_errors  = len(results["errors"])
+    num_success = len(results["success"])
+    num_total   = num_errors + num_success
+
+    print(f"Errors:  {num_errors}/{num_total}")
+    print(f"Success: {num_success}/{num_total}")
+    # print(results)
+
+    if (num_errors/num_total) > (num_success/num_total):
+        exit(1)