Mystery test suite

.gitignore

@@ -36,6 +36,6 @@ resources/user/*
 get-pip.py
 
 venv
-test
+test_games/
 data/sprites/official/selan.1.zspr
 *.zspr

Mystery.py

@@ -28,6 +28,7 @@ def main():
     parser.add_argument('--names', default='')
     parser.add_argument('--teams', default=1, type=lambda value: max(int(value), 1))
     parser.add_argument('--create_spoiler', action='store_true')
+    parser.add_argument('--suppress_rom', action='store_true')
     parser.add_argument('--rom')
     parser.add_argument('--enemizercli')
     parser.add_argument('--outputpath')
@@ -61,6 +62,7 @@ def main():
     erargs.seed = seed
     erargs.names = args.names
     erargs.create_spoiler = args.create_spoiler
+    erargs.suppress_rom = args.suppress_rom
     erargs.race = True
     erargs.outputname = seedname
     erargs.outputpath = args.outputpath

mystery_testsuite.yml

@@ -0,0 +1,164 @@
+description: A test suite for testing various combinations
+# Not yet in this branch
+#algorithm:
+#  major_only: 1
+#  dungeon_only: 1
+#  vanilla_fill: 1
+#  balanced: 10
+#  district: 1
+door_shuffle:
+  vanilla: 1
+  basic: 2
+  crossed: 3  # crossed yield more errors so is preferred
+intensity:
+  1: 1
+  2: 1
+  3: 2  # intensity 3 usuall yield more errors
+keydropshuffle:
+  on: 1
+  off: 1
+shopsanity:
+  on: 1
+  off: 1
+pot_shuffle:
+  on: 1
+  off: 1
+entrance_shuffle:
+  none: 1
+  dungeonssimple: 1
+  dungeonsfull: 1
+  simple: 1
+  restricted: 1
+  full: 1
+  crossed: 1
+  insanity: 1
+shufflelinks:
+  on: 1
+  off: 1
+world_state:
+  standard: 1
+  open: 1
+  inverted: 1
+  retro: 0
+retro:
+  on: 1
+  off: 1
+goals:
+  ganon: 1
+  fast_ganon: 1
+  dungeons: 2  # this yields more errors so is preferred
+  pedestal: 1
+  triforce-hunt: 1
+triforce_goal_min: 20
+triforce_goal_max: 30
+triforce_pool_min: 30
+triforce_pool_max: 40
+triforce_min_difference: 10
+map_shuffle:
+  on: 1
+  off: 1
+compass_shuffle:
+  on: 1
+  off: 1
+smallkey_shuffle:
+  on: 1
+  off: 1
+bigkey_shuffle:
+  on: 1
+  off: 1
+dungeon_counters:
+  on: 1
+  off: 1
+  default: 1
+experimental:
+  on: 1
+  off: 1
+glitches_required:
+  none: 10  # i'm more interest in testing shuffles with more restrictive logic
+  owg: 1
+  no_logic: 1
+accessibility:
+  items: 1
+  locations: 1
+  none: 0  # i'm not really interested in this yet
+restrict_boss_items:
+  none: 1
+  mapcompass: 1
+  dungeon: 1
+tower_open:
+  "0": 1
+  "1": 1
+  "2": 1
+  "3": 1
+  "4": 1
+  "5": 1
+  "6": 1
+  "7": 10  # more restrictions is usually best for testing
+  random: 1
+ganon_open:
+  "0": 1
+  "1": 1
+  "2": 1
+  "3": 1
+  "4": 1
+  "5": 1
+  "6": 1
+  "7": 10 # more restrictions is usually best for testing
+  random: 1
+boss_shuffle:
+  none: 1
+  simple: 1
+  full: 1
+  random: 1
+enemy_shuffle:  # shouldn't affect generation
+  none: 1
+  shuffled: 1
+  random: 1
+  legacy: 0
+hints:
+  on: 1
+  off: 1
+pseudoboots:  # shouldn't affect generation
+  on: 1
+  off: 1
+weapons:
+  randomized: 1
+  assured: 1
+  vanilla: 1
+  swordless: 1
+item_pool:
+  normal: 1
+  hard: 1
+  expert: 1
+item_functionality: # shouldn't affect generation
+  normal: 1
+  hard: 0
+  expert: 0
+enemy_damage: # shouldn't affect generation
+  default: 1
+  shuffled: 0
+  random: 0
+enemy_health: # shouldn't affect generation
+  default: 1
+  easy: 0
+  hard: 0
+  expert: 0
+rom:
+  quickswap:  # shouldn't affect generation
+    on: 1
+    off: 0
+# reduce_flashing: should affect generation at this point
+  heartcolor:  # shouldn't affect generation
+    red: 1
+    blue: 1
+    green: 1
+    yellow: 1
+  heartbeep:  # shouldn't affect generation
+    double: 0
+    normal: 0
+    half: 0
+    quarter: 1
+    off: 0
+  shuffle_sfx:
+    on: 1
+    off: 1

source/test/MysteryTestSuite.py

@@ -0,0 +1,124 @@
+import subprocess
+import sys
+import multiprocessing
+import concurrent.futures
+import argparse
+from collections import OrderedDict
+
+cpu_threads = multiprocessing.cpu_count()
+py_version = f"{sys.version_info.major}.{sys.version_info.minor}"
+
+
+def main(args=None):
+    successes = []
+    errors = []
+    task_mapping = []
+    tests = OrderedDict()
+
+    successes.append(f"Testing {args.dr} DR with {args.count} Tests" + (f" (intensity={args.tense})" if args.dr in ['basic', 'crossed'] else ""))
+    print(successes[0])
+
+    max_attempts = args.count
+    pool = concurrent.futures.ThreadPoolExecutor(max_workers=cpu_threads)
+    dead_or_alive = 0
+    alive = 0
+
+    def test(testname: str, command: str):
+        tests[testname] = [command]
+        basecommand = f"python3.8 Mystery.py --suppress_rom"
+
+        def gen_seed():
+            taskcommand = basecommand + " " + command
+            return subprocess.run(taskcommand, capture_output=True, shell=True, text=True)
+
+        for x in range(1, max_attempts + 1):
+            task = pool.submit(gen_seed)
+            task.success = False
+            task.name = testname
+            task.mode = "Mystery"
+            task.cmd = basecommand + " " + command
+            task_mapping.append(task)
+
+    for i in range(0, 100):
+        test("Mystery", "--weights mystery_testsuite.yml")
+
+    from tqdm import tqdm
+    with tqdm(concurrent.futures.as_completed(task_mapping),
+              total=len(task_mapping), unit="seed(s)",
+              desc=f"Success rate: 0.00%") as progressbar:
+        for task in progressbar:
+            dead_or_alive += 1
+            try:
+                result = task.result()
+                if result.returncode:
+                    errors.append([task.name, task.cmd, result.stderr])
+                else:
+                    alive += 1
+                    task.success = True
+            except Exception as e:
+                raise e
+
+            progressbar.set_description(f"Success rate: {(alive/dead_or_alive)*100:.2f}% - {task.name}")
+
+    def get_results(testname: str):
+        result = ""
+        for mode in ['Mystery']:
+            dead_or_alive = [task.success for task in task_mapping if task.name == testname and task.mode == mode]
+            alive = [x for x in dead_or_alive if x]
+            success = f"{testname} Rate: {(len(alive) / len(dead_or_alive)) * 100:.2f}%"
+            successes.append(success)
+            print(success)
+            result += f"{(len(alive)/len(dead_or_alive))*100:.2f}%\t"
+        return result.strip()
+
+    results = []
+    for t in tests.keys():
+        results.append(get_results(t))
+
+    for result in results:
+        print(result)
+        successes.append(result)
+
+    return successes, errors
+
+
+if __name__ == "__main__":
+    successes = []
+
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument('--count', default=0, type=lambda value: max(int(value), 0))
+    parser.add_argument('--cpu_threads', default=cpu_threads, type=lambda value: max(int(value), 1))
+    parser.add_argument('--help', default=False, action='store_true')
+
+    args = parser.parse_args()
+
+    if args.help:
+        parser.print_help()
+        exit(0)
+
+    cpu_threads = args.cpu_threads
+
+    for dr in [['mystery', args.count if args.count else 1, 1]]:
+
+        for tense in range(1, dr[2] + 1):
+            args = argparse.Namespace()
+            args.dr = dr[0]
+            args.tense = tense
+            args.count = dr[1]
+            s, errors = main(args=args)
+            if successes:
+                successes += [""] * 2
+            successes += s
+            print()
+
+            if errors:
+                with open(f"{dr[0]}{(f'-{tense}' if dr[0] in ['basic', 'crossed'] else '')}-errors.txt", 'w') as stream:
+                    for error in errors:
+                        stream.write(error[0] + "\n")
+                        stream.write(error[1] + "\n")
+                        stream.write(error[2] + "\n\n")
+
+    with open("success.txt", "w") as stream:
+        stream.write(str.join("\n", successes))
+
+    input("Press enter to continue")