From b21564d5aa738c0174a070434a71b7dcb1299fc2 Mon Sep 17 00:00:00 2001
From: aerinon <randall.rupper@sirsidynix.com>
Date: Tue, 29 Jun 2021 16:34:28 -0600
Subject: [PATCH] Combinatoric approach revised (KLA1) Backported some fixes

---
 BaseClasses.py    | 359 +++++++++++++++++++++++++++++++++++++++-------
 DoorShuffle.py    |  35 +----
 Dungeons.py       |  15 --
 Fill.py           |  16 ++-
 KeyDoorShuffle.py | 146 +++++++++++++------
 Main.py           |   1 +
 Rules.py          |  52 +++++--
 Utils.py          |  24 ++++
 8 files changed, 490 insertions(+), 158 deletions(-)

diff --git a/BaseClasses.py b/BaseClasses.py
index ba685ac2..54a45817 100644
--- a/BaseClasses.py
+++ b/BaseClasses.py
@@ -458,17 +458,26 @@ class World(object):
 
 class CollectionState(object):
 
-    def __init__(self, parent):
-        self.prog_items = Counter()
+    def __init__(self, parent, skip_init=False):
         self.world = parent
-        self.reachable_regions = {player: dict() for player in range(1, parent.players + 1)}
-        self.blocked_connections = {player: dict() for player in range(1, parent.players + 1)}
-        self.events = []
-        self.path = {}
-        self.locations_checked = set()
-        self.stale = {player: True for player in range(1, parent.players + 1)}
-        for item in parent.precollected_items:
-            self.collect(item, True)
+        if not skip_init:
+            self.prog_items = Counter()
+            self.reachable_regions = {player: dict() for player in range(1, parent.players + 1)}
+            self.blocked_connections = {player: dict() for player in range(1, parent.players + 1)}
+            self.events = []
+            self.path = {}
+            self.locations_checked = set()
+            self.stale = {player: True for player in range(1, parent.players + 1)}
+            for item in parent.precollected_items:
+                self.collect(item, True)
+            # reached vs. opened in the counter
+            self.door_counter = {player: (Counter(), Counter()) for player in range(1, parent.players + 1)}
+            self.reached_doors = {player: set() for player in range(1, parent.players + 1)}
+            self.opened_doors = {player: set() for player in range(1, parent.players + 1)}
+            self.dungeons_to_check = {player: defaultdict(dict) for player in range(1, parent.players + 1)}
+
+            self.ghost_keys = Counter()
+        self.dungeon_limits = None
 
     def update_reachable_regions(self, player):
         self.stale[player] = False
@@ -479,66 +488,261 @@ class CollectionState(object):
         start = self.world.get_region('Menu', player)
         if not start in rrp:
             rrp[start] = CrystalBarrier.Orange
-            for exit in start.exits:
-                bc[exit] = CrystalBarrier.Orange
+            for conn in start.exits:
+                bc[conn] = CrystalBarrier.Orange
 
         queue = deque(self.blocked_connections[player].items())
 
+        self.traverse_world(queue, rrp, bc, player)
+        unresolved_events = [x for y in self.reachable_regions[player] for x in y.locations
+                             if x.event and x.item and (x.item.smallkey or x.item.bigkey or x.item.advancement)
+                             and x not in self.locations_checked and x.can_reach(self)]
+        if len(unresolved_events) == 0:
+            self.check_key_doors_in_dungeons(rrp, player)
+
+    def traverse_world(self, queue, rrp, bc, player):
         # run BFS on all connections, and keep track of those blocked by missing items
-        while True:
-            try:
-                connection, crystal_state = queue.popleft()
-                new_region = connection.connected_region
-                if new_region is None or new_region in rrp and (new_region.type != RegionType.Dungeon or (rrp[new_region] & crystal_state) == crystal_state):
-                    bc.pop(connection, None)
-                elif connection.can_reach(self):
-                    if new_region.type == RegionType.Dungeon:
-                        new_crystal_state = crystal_state
-                        for exit in new_region.exits:
-                            door = exit.door
-                            if door is not None and door.crystal == CrystalBarrier.Either and door.entrance.can_reach(self):
-                                new_crystal_state = CrystalBarrier.Either
-                                break
-                        if new_region in rrp:
-                            new_crystal_state |= rrp[new_region]
+        while len(queue) > 0:
+            connection, crystal_state = queue.popleft()
+            new_region = connection.connected_region
+            if not self.should_visit(new_region, rrp, crystal_state, player):
+                bc.pop(connection, None)
+            elif connection.can_reach(self):
+                bc.pop(connection, None)
+                if new_region.type == RegionType.Dungeon:
+                    new_crystal_state = crystal_state
+                    if new_region in rrp:
+                        new_crystal_state |= rrp[new_region]
 
-                        rrp[new_region] = new_crystal_state
-
-                        for exit in new_region.exits:
-                            door = exit.door
-                            if door is not None and not door.blocked:
+                    rrp[new_region] = new_crystal_state
+                    for conn in new_region.exits:
+                        door = conn.door
+                        if door is not None and not door.blocked:
+                            if self.valid_crystal(door, new_crystal_state):
                                 door_crystal_state = door.crystal if door.crystal else new_crystal_state
-                                bc[exit] = door_crystal_state
-                                queue.append((exit, door_crystal_state))
-                            elif door is None:
-                                queue.append((exit, new_crystal_state))
+                                bc[conn] = door_crystal_state
+                                queue.append((conn, door_crystal_state))
+                        elif door is None:
+                            # note: no door in dungeon indicates what exactly? (always traversable)?
+                            queue.append((conn, new_crystal_state))
+                else:
+                    new_crystal_state = CrystalBarrier.Orange
+                    rrp[new_region] = new_crystal_state
+                    for conn in new_region.exits:
+                        bc[conn] = new_crystal_state
+                        queue.append((conn, new_crystal_state))
+
+                self.path[new_region] = (new_region.name, self.path.get(connection, None))
+
+                # Retry connections if the new region can unblock them
+                if new_region.name in indirect_connections:
+                    new_entrance = self.world.get_entrance(indirect_connections[new_region.name], player)
+                    if new_entrance in bc and new_entrance.parent_region in rrp:
+                        new_crystal_state = rrp[new_entrance.parent_region]
+                        if (new_entrance, new_crystal_state) not in queue:
+                            queue.append((new_entrance, new_crystal_state))
+            # else those connections that are not accessible yet
+            if self.is_small_door(connection) and not self.world.retro[player]:  # todo: retro
+                door = connection.door
+                dungeon_name = connection.parent_region.dungeon.name  # todo: universal
+                key_logic = self.world.key_logic[player][dungeon_name]
+                if door.name not in self.reached_doors[player]:
+                    self.door_counter[player][0][dungeon_name] += 1
+                    self.reached_doors[player].add(door.name)
+                    if key_logic.sm_doors[door]:
+                        self.reached_doors[player].add(key_logic.sm_doors[door].name)
+                if not connection.can_reach(self):
+                    checklist = self.dungeons_to_check[player][dungeon_name]
+                    checklist[connection.name] = (connection, crystal_state)
+                elif door.name not in self.opened_doors[player]:
+                    opened_doors = self.opened_doors[player]
+                    door = connection.door
+                    if door.name not in opened_doors:
+                        self.door_counter[player][1][dungeon_name] += 1
+                        opened_doors.add(door.name)
+                        key_logic = self.world.key_logic[player][dungeon_name]
+                        if key_logic.sm_doors[door]:
+                            opened_doors.add(key_logic.sm_doors[door].name)
+
+    def should_visit(self, new_region, rrp, crystal_state, player):
+        if not new_region:
+            return False
+        if self.dungeon_limits and not self.possibly_connected_to_dungeon(new_region, player):
+            return False
+        if new_region not in rrp:
+            return True
+        if new_region.type != RegionType.Dungeon:
+            return False
+        return (rrp[new_region] & crystal_state) != crystal_state
+
+    def possibly_connected_to_dungeon(self, new_region, player):
+        if new_region.dungeon:
+            return new_region.dungeon.name in self.dungeon_limits
+        else:
+            return new_region.name in self.world.inaccessible_regions[player]
+
+    @staticmethod
+    def valid_crystal(door, new_crystal_state):
+        return (not door.crystal or door.crystal == CrystalBarrier.Either or new_crystal_state == CrystalBarrier.Either
+                or new_crystal_state == door.crystal)
+
+    def check_key_doors_in_dungeons(self, rrp, player):
+        for dungeon_name, checklist in self.dungeons_to_check[player].items():
+            init_door_candidates = self.should_explore_child_state(self, dungeon_name, player)
+            key_total = self.prog_items[(dungeon_keys[dungeon_name], player)]  # todo: universal
+            remaining_keys = key_total - self.door_counter[player][1][dungeon_name]
+            if not init_door_candidates or remaining_keys == 0:
+                continue
+            dungeon_doors = {x.name for x in self.world.key_logic[player][dungeon_name].sm_doors.keys()}
+
+            def valid_d_door(x):
+                return x in dungeon_doors
+
+            child_states = deque()
+            child_states.append(self)
+            visited_opened_doors = set()
+            visited_opened_doors.add(frozenset(self.opened_doors[player]))
+            terminal_states, done, common_regions, common_bc, common_doors = [], False, {}, {}, set()
+            while not done:
+                terminal_states.clear()
+                while len(child_states) > 0:
+                    next_child = child_states.popleft()
+                    door_candidates = CollectionState.should_explore_child_state(next_child, dungeon_name, player)
+                    if door_candidates:
+                        for chosen_door in door_candidates:
+                            child_state = next_child.copy()
+                            child_queue = deque()
+                            child_state.door_counter[player][1][dungeon_name] += 1
+                            if isinstance(chosen_door, tuple):
+                                child_state.opened_doors[player].add(chosen_door[0])
+                                child_state.opened_doors[player].add(chosen_door[1])
+                                if chosen_door[0] in checklist:
+                                    child_queue.append(checklist[chosen_door[0]])
+                                if chosen_door[1] in checklist:
+                                    child_queue.append(checklist[chosen_door[1]])
+                            else:
+                                child_state.opened_doors[player].add(chosen_door)
+                                if chosen_door in checklist:
+                                    child_queue.append(checklist[chosen_door])
+                            if child_state.opened_doors[player] not in visited_opened_doors:
+                                done = False
+                                while not done:
+                                    rrp_ = child_state.reachable_regions[player]
+                                    bc_ = child_state.blocked_connections[player]
+                                    self.dungeon_limits = [dungeon_name]
+                                    child_state.traverse_world(child_queue, rrp_, bc_, player)
+                                    new_events = child_state.sweep_for_events_once()
+                                    child_state.stale[player] = False
+                                    if new_events:
+                                        for conn in bc_:
+                                            if conn.parent_region.dungeon and conn.parent_region.dungeon.name == dungeon_name:
+                                                child_queue.append((conn, bc_[conn]))
+                                    done = not new_events
+                                visited_opened_doors.add(frozenset(child_state.opened_doors[player]))
+                                child_states.append(child_state)
                     else:
-                        new_crystal_state = CrystalBarrier.Orange
-                        rrp[new_region] = new_crystal_state
-                        bc.pop(connection, None)
-                        for exit in new_region.exits:
-                            bc[exit] = new_crystal_state
-                            queue.append((exit, new_crystal_state))
+                        terminal_states.append(next_child)
+                common_regions, common_doors, first = {}, set(), True
+                for term_state in terminal_states:
+                    t_rrp = term_state.reachable_regions[player]
+                    if first:
+                        first = False
+                        common_regions = {x: y for x, y in t_rrp.items() if x not in rrp or y != rrp[x]}
+                        common_doors = {x for x in term_state.opened_doors[player] - self.opened_doors[player]
+                                        if valid_d_door(x)}
+                    else:
+                        cm_rrp = {x: y for x, y in t_rrp.items() if x not in rrp or y != rrp[x]}
+                        common_regions = {k: self.comb_crys(v, cm_rrp[k]) for k, v in common_regions.items()
+                                          if k in cm_rrp and self.crys_agree(v, cm_rrp[k])}
+                        common_doors &= {x for x in term_state.opened_doors[player] - self.opened_doors[player]
+                                         if valid_d_door(x)}
+                done = len(child_states) == 0
 
-                    self.path[new_region] = (new_region.name, self.path.get(connection, None))
+            terminal_queue = deque()
+            for door in common_doors:
+                self.opened_doors[player].add(door)
+                if door in checklist:
+                    terminal_queue.append(checklist[door])
+                if self.find_door_pair(player, dungeon_name, door) not in self.opened_doors[player]:
+                    self.door_counter[player][1][dungeon_name] += 1
 
-                    # Retry connections if the new region can unblock them
-                    if new_region.name in indirect_connections:
-                        new_entrance = self.world.get_entrance(indirect_connections[new_region.name], player)
-                        if new_entrance in bc and new_entrance not in queue and new_entrance.parent_region in rrp:
-                            queue.append((new_entrance, rrp[new_entrance.parent_region]))
-            except IndexError:
-                break
+            self.dungeon_limits = [dungeon_name]
+            rrp_ = self.reachable_regions[player]
+            bc_ = self.blocked_connections[player]
+            self.traverse_world(terminal_queue, rrp_, bc_, player)
+            self.dungeon_limits = None
 
+            rrp = self.reachable_regions[player]
+            missing_regions = {x: y for x, y in common_regions.items() if x not in rrp}
+            for k in missing_regions:
+                rrp[k] = missing_regions[k]
+            checklist.clear()
+
+    @staticmethod
+    def comb_crys(a, b):
+        return a if a == b or a != CrystalBarrier.Either else b
+
+    @staticmethod
+    def crys_agree(a, b):
+        return a == b or a == CrystalBarrier.Either or b == CrystalBarrier.Either
+
+    def find_door_pair(self, player, dungeon_name, name):
+        for door in self.world.key_logic[player][dungeon_name].sm_doors.keys():
+            if door.name == name:
+                paired_door = self.world.key_logic[player][dungeon_name].sm_doors[door]
+                return paired_door.name if paired_door else None
+        return None
+
+    @staticmethod
+    def should_explore_child_state(state, dungeon_name, player):
+        small_key_name = dungeon_keys[dungeon_name]  # todo: universal
+        key_total = state.prog_items[(small_key_name, player)] + state.ghost_keys[(small_key_name, player)]
+        remaining_keys = key_total - state.door_counter[player][1][dungeon_name]
+        unopened_doors = state.door_counter[player][0][dungeon_name] - state.door_counter[player][1][dungeon_name]
+        if remaining_keys > 0 and unopened_doors > 0:
+            key_logic = state.world.key_logic[player][dungeon_name]   # todo: universal
+            door_candidates, skip = [], set()
+            for door, paired in key_logic.sm_doors.items():
+                if door.name in state.reached_doors[player] and door.name not in state.opened_doors[player]:
+                    if door.name not in skip:
+                        if paired:
+                            door_candidates.append((door.name, paired.name))
+                            skip.add(paired.name)
+                        else:
+                            door_candidates.append(door.name)
+            return door_candidates
+        return None
+
+    @staticmethod
+    def print_rrp(rrp):
+        logger = logging.getLogger('')
+        logger.debug('RRP Checking')
+        for region, packet in rrp.items():
+            new_crystal_state, logic, path = packet
+            logger.debug(f'\nRegion: {region.name} (CS: {str(new_crystal_state)})')
+            for i in range(0, len(logic)):
+                logger.debug(f'{logic[i]}')
+                logger.debug(f'{",".join(str(x) for x in path[i])}')
 
     def copy(self):
-        ret = CollectionState(self.world)
+        ret = CollectionState(self.world, skip_init=True)
         ret.prog_items = self.prog_items.copy()
         ret.reachable_regions = {player: copy.copy(self.reachable_regions[player]) for player in range(1, self.world.players + 1)}
         ret.blocked_connections = {player: copy.copy(self.blocked_connections[player]) for player in range(1, self.world.players + 1)}
         ret.events = copy.copy(self.events)
         ret.path = copy.copy(self.path)
         ret.locations_checked = copy.copy(self.locations_checked)
+        ret.stale = {player: self.stale[player] for player in range(1, self.world.players + 1)}
+        ret.door_counter = {player: (copy.copy(self.door_counter[player][0]), copy.copy(self.door_counter[player][1]))
+                            for player in range(1, self.world.players + 1)}
+        ret.reached_doors = {player: copy.copy(self.reached_doors[player]) for player in range(1, self.world.players + 1)}
+        ret.opened_doors = {player: copy.copy(self.opened_doors[player]) for player in range(1, self.world.players + 1)}
+        # todo: verify if this isn't copied deep enough
+        ret.dungeons_to_check = {
+            player: defaultdict(dict, {name: copy.copy(checklist)
+                                       for name, checklist in self.dungeons_to_check[player].items()})
+            for player in range(1, self.world.players + 1)}
+        ret.ghost_keys = self.ghost_keys.copy()
         return ret
 
     def can_reach(self, spot, resolution_hint=None, player=None):
@@ -556,6 +760,19 @@ class CollectionState(object):
 
         return spot.can_reach(self)
 
+    def sweep_for_events_once(self, key_only=False, locations=None):
+        if locations is None:
+            locations = self.world.get_filled_locations()
+        checked_locations = set([l for l in locations if l in self.locations_checked])
+        reachable_events = [location for location in locations if location.event and
+                            (not key_only or (not self.world.keyshuffle[location.item.player] and location.item.smallkey) or (not self.world.bigkeyshuffle[location.item.player] and location.item.bigkey))
+                            and location.can_reach(self)]
+        reachable_events = self._do_not_flood_the_keys(reachable_events)
+        for event in reachable_events:
+            if event not in checked_locations:
+                self.events.append((event.name, event.player))
+                self.collect(event.item, True, event)
+        return len(reachable_events) > len(checked_locations)
 
     def sweep_for_events(self, key_only=False, locations=None):
         # this may need improvement
@@ -603,6 +820,13 @@ class CollectionState(object):
                     or not self.location_can_be_flooded(flood_location))
         return True
 
+    @staticmethod
+    def is_small_door(connection):
+        return connection and connection.door and connection.door.smallKey
+
+    def is_door_open(self, door_name, player):
+        return door_name in self.opened_doors[player]
+
     @staticmethod
     def location_can_be_flooded(location):
         return location.parent_region.name in ['Swamp Trench 1 Alcove', 'Swamp Trench 2 Alcove']
@@ -1806,6 +2030,15 @@ class Item(object):
     def compass(self):
         return self.type == 'Compass'
 
+    @property
+    def dungeon(self):
+        if not self.smallkey and not self.bigkey and not self.map and not self.compass:
+            return None
+        item_dungeon = self.name.split('(')[1][:-1]
+        if item_dungeon == 'Escape':
+            item_dungeon = 'Hyrule Castle'
+        return item_dungeon
+
     def __str__(self):
         return str(self.__unicode__())
 
@@ -2196,6 +2429,21 @@ dungeon_names = [
     'Swamp Palace', 'Skull Woods', 'Thieves Town', 'Ice Palace', 'Misery Mire', 'Turtle Rock', 'Ganons Tower'
 ]
 
+dungeon_keys = {
+    'Hyrule Castle': 'Small Key (Escape)',
+    'Eastern Palace': 'Small Key (Eastern Palace)',
+    'Desert Palace': 'Small Key (Desert Palace)',
+    'Tower of Hera': 'Small Key (Tower of Hera)',
+    'Agahnims Tower': 'Small Key (Agahnims Tower)',
+    'Palace of Darkness': 'Small Key (Palace of Darkness)',
+    'Swamp Palace': 'Small Key (Swamp Palace)',
+    'Skull Woods': 'Small Key (Skull Woods)',
+    'Thieves Town': 'Small Key (Thieves Town)',
+    'Ice Palace': 'Small Key (Ice Palace)',
+    'Misery Mire': 'Small Key (Misery Mire)',
+    'Turtle Rock': 'Small Key (Turtle Rock)',
+    'Ganons Tower': 'Small Key (Ganons Tower)'
+}
 
 class PotItem(FastEnum):
     Nothing = 0x0
@@ -2346,3 +2594,10 @@ class Settings(object):
         args.enemy_health[p] = r(e_health)[(settings[7] & 0xE0) >> 5]
         args.enemy_damage[p] = r(e_dmg)[(settings[7] & 0x18) >> 3]
         args.shufflepots[p] = True if settings[7] & 0x4 else False
+
+
+@unique
+class KeyRuleType(Enum):
+    WorstCase = 0
+    AllowSmall = 1
+    Lock = 2
diff --git a/DoorShuffle.py b/DoorShuffle.py
index ee38f00c..53676cb9 100644
--- a/DoorShuffle.py
+++ b/DoorShuffle.py
@@ -1,22 +1,21 @@
 import random
 from collections import defaultdict, deque
 import logging
-import operator as op
 import time
 from enum import unique, Flag
 from typing import DefaultDict, Dict, List
 
-from functools import reduce
-from BaseClasses import RegionType, Region, Door, DoorType, Direction, Sector, CrystalBarrier, DungeonInfo
+from BaseClasses import RegionType, Region, Door, DoorType, Direction, Sector, CrystalBarrier, DungeonInfo, dungeon_keys
 from Doors import reset_portals
 from Dungeons import dungeon_regions, region_starts, standard_starts, split_region_starts
-from Dungeons import dungeon_bigs, dungeon_keys, dungeon_hints
+from Dungeons import dungeon_bigs, dungeon_hints
 from Items import ItemFactory
 from RoomData import DoorKind, PairedDoor, reset_rooms
 from DungeonGenerator import ExplorationState, convert_regions, generate_dungeon, pre_validate, determine_required_paths, drop_entrances
 from DungeonGenerator import create_dungeon_builders, split_dungeon_builder, simple_dungeon_builder, default_dungeon_entrances
 from DungeonGenerator import dungeon_portals, dungeon_drops, GenerationException
-from KeyDoorShuffle import analyze_dungeon, validate_vanilla_key_logic, build_key_layout, validate_key_layout
+from KeyDoorShuffle import analyze_dungeon, build_key_layout, validate_key_layout
+from Utils import ncr, kth_combination
 
 
 def link_doors(world, player):
@@ -212,8 +211,8 @@ def vanilla_key_logic(world, player):
         analyze_dungeon(key_layout, world, player)
         world.key_logic[player][builder.name] = key_layout.key_logic
         log_key_logic(builder.name, key_layout.key_logic)
-    if world.shuffle[player] == 'vanilla' and world.accessibility[player] == 'items' and not world.retro[player] and not world.keydropshuffle[player]:
-        validate_vanilla_key_logic(world, player)
+    # if world.shuffle[player] == 'vanilla' and world.accessibility[player] == 'items' and not world.retro[player] and not world.keydropshuffle[player]:
+    #     validate_vanilla_key_logic(world, player)
 
 
 # some useful functions
@@ -1576,28 +1575,6 @@ def find_key_door_candidates(region, checked, world, player):
     return candidates, checked_doors
 
 
-def kth_combination(k, l, r):
-    if r == 0:
-        return []
-    elif len(l) == r:
-        return l
-    else:
-        i = ncr(len(l)-1, r-1)
-        if k < i:
-            return l[0:1] + kth_combination(k, l[1:], r-1)
-        else:
-            return kth_combination(k-i, l[1:], r)
-
-
-def ncr(n, r):
-    if r == 0:
-        return 1
-    r = min(r, n-r)
-    numerator = reduce(op.mul, range(n, n-r, -1), 1)
-    denominator = reduce(op.mul, range(1, r+1), 1)
-    return numerator / denominator
-
-
 def reassign_key_doors(builder, world, player):
     logger = logging.getLogger('')
     logger.debug('Key doors for %s', builder.name)
diff --git a/Dungeons.py b/Dungeons.py
index d5297400..6f0f2197 100644
--- a/Dungeons.py
+++ b/Dungeons.py
@@ -375,21 +375,6 @@ flexible_starts = {
     'Skull Woods': ['Skull Left Drop', 'Skull Pinball']
 }
 
-dungeon_keys = {
-    'Hyrule Castle': 'Small Key (Escape)',
-    'Eastern Palace': 'Small Key (Eastern Palace)',
-    'Desert Palace': 'Small Key (Desert Palace)',
-    'Tower of Hera': 'Small Key (Tower of Hera)',
-    'Agahnims Tower': 'Small Key (Agahnims Tower)',
-    'Palace of Darkness': 'Small Key (Palace of Darkness)',
-    'Swamp Palace': 'Small Key (Swamp Palace)',
-    'Skull Woods': 'Small Key (Skull Woods)',
-    'Thieves Town': 'Small Key (Thieves Town)',
-    'Ice Palace': 'Small Key (Ice Palace)',
-    'Misery Mire': 'Small Key (Misery Mire)',
-    'Turtle Rock': 'Small Key (Turtle Rock)',
-    'Ganons Tower': 'Small Key (Ganons Tower)'
-}
 
 dungeon_bigs = {
     'Hyrule Castle': 'Big Key (Escape)',
diff --git a/Fill.py b/Fill.py
index c1488113..9c914726 100644
--- a/Fill.py
+++ b/Fill.py
@@ -199,6 +199,7 @@ def fill_restrictive(world, base_state, locations, itempool, keys_in_itempool =
 
                 spot_to_fill = None
 
+                valid_locations = []
                 for location in locations:
                     if item_to_place.smallkey or item_to_place.bigkey:  # a better test to see if a key can go there
                         location.item = item_to_place
@@ -209,11 +210,16 @@ def fill_restrictive(world, base_state, locations, itempool, keys_in_itempool =
                     if (not single_player_placement or location.player == item_to_place.player)\
                             and location.can_fill(test_state, item_to_place, perform_access_check)\
                             and valid_key_placement(item_to_place, location, itempool if (keys_in_itempool and keys_in_itempool[item_to_place.player]) else world.itempool, world):
-                        spot_to_fill = location
-                        break
-                    elif item_to_place.smallkey or item_to_place.bigkey:
+                        # todo: optimization: break instead of cataloging all valid locations
+                        if not spot_to_fill:
+                            spot_to_fill = location
+                        valid_locations.append(location)
+
+                    if item_to_place.smallkey or item_to_place.bigkey:
                         location.item = None
 
+                logging.getLogger('').debug(f'{item_to_place} valid placement at {len(valid_locations)} locations')
+
                 if spot_to_fill is None:
                     # we filled all reachable spots. Maybe the game can be beaten anyway?
                     unplaced_items.insert(0, item_to_place)
@@ -250,9 +256,7 @@ def valid_key_placement(item, location, itempool, world):
 def track_outside_keys(item, location, world):
     if not item.smallkey:
         return
-    item_dungeon = item.name.split('(')[1][:-1]
-    if item_dungeon == 'Escape':
-        item_dungeon = 'Hyrule Castle'
+    item_dungeon = item.dungeon
     if location.player == item.player:
         loc_dungeon = location.parent_region.dungeon
         if loc_dungeon and loc_dungeon.name == item_dungeon:
diff --git a/KeyDoorShuffle.py b/KeyDoorShuffle.py
index 549a739c..857ee479 100644
--- a/KeyDoorShuffle.py
+++ b/KeyDoorShuffle.py
@@ -2,9 +2,9 @@ import itertools
 import logging
 from collections import defaultdict, deque
 
-from BaseClasses import DoorType
+from BaseClasses import DoorType, dungeon_keys, KeyRuleType
 from Regions import dungeon_events
-from Dungeons import dungeon_keys, dungeon_bigs
+from Dungeons import dungeon_bigs
 from DungeonGenerator import ExplorationState, special_big_key_doors
 
 
@@ -25,6 +25,7 @@ class KeyLayout(object):
         self.all_locations = set()
         self.item_locations = set()
 
+        self.found_doors = set()
         # bk special?
         # bk required? True if big chests or big doors exists
 
@@ -54,6 +55,7 @@ class KeyLogic(object):
         self.location_rules = {}
         self.outside_keys = 0
         self.dungeon = dungeon_name
+        self.sm_doors = {}
 
     def check_placement(self, unplaced_keys, big_key_loc=None):
         for rule in self.placement_rules:
@@ -65,6 +67,15 @@ class KeyLogic(object):
                         return False
         return True
 
+    def reset(self):
+        self.door_rules.clear()
+        self.bk_restricted.clear()
+        self.bk_locked.clear()
+        self.sm_restricted.clear()
+        self.bk_doors.clear()
+        self.bk_chests.clear()
+        self.placement_rules.clear()
+
 
 class DoorRules(object):
 
@@ -79,6 +90,8 @@ class DoorRules(object):
         self.small_location = None
         self.opposite = None
 
+        self.new_rules = {}  # keyed by type, or type+lock_item -> number
+
 
 class LocationRule(object):
     def __init__(self):
@@ -209,8 +222,19 @@ def calc_max_chests(builder, key_layout, world, player):
 
 
 def analyze_dungeon(key_layout, world, player):
+    key_layout.key_logic.reset()
     key_layout.key_counters = create_key_counters(key_layout, world, player)
     key_logic = key_layout.key_logic
+    for door in key_layout.proposal:
+        if isinstance(door, tuple):
+            key_logic.sm_doors[door[0]] = door[1]
+            key_logic.sm_doors[door[1]] = door[0]
+        else:
+            if door.dest and door.type != DoorType.SpiralStairs:
+                key_logic.sm_doors[door] = door.dest
+                key_logic.sm_doors[door.dest] = door
+            else:
+                key_logic.sm_doors[door] = None
 
     find_bk_locked_sections(key_layout, world, player)
     key_logic.bk_chests.update(find_big_chest_locations(key_layout.all_chest_locations))
@@ -247,8 +271,9 @@ def analyze_dungeon(key_layout, world, player):
         while len(child_queue) > 0:
             child, odd_counter, empty_flag = child_queue.popleft()
             if not child.bigKey and child not in doors_completed:
-                best_counter = find_best_counter(child, odd_counter, key_counter, key_layout, world, player, False, empty_flag)
-                rule = create_rule(best_counter, key_counter, key_layout, world, player)
+                best_counter = find_best_counter(child, key_layout, odd_counter, False, empty_flag)
+                rule = create_rule(best_counter, key_counter, world, player)
+                create_worst_case_rule(rule, best_counter, world, player)
                 check_for_self_lock_key(rule, child, best_counter, key_layout, world, player)
                 bk_restricted_rules(rule, child, odd_counter, empty_flag, key_counter, key_layout, world, player)
                 key_logic.door_rules[child.name] = rule
@@ -258,7 +283,8 @@ def analyze_dungeon(key_layout, world, player):
             if ctr_id not in visited_cid:
                 queue.append((child, next_counter))
                 visited_cid.add(ctr_id)
-    check_rules(original_key_counter, key_layout, world, player)
+    # todo: why is this commented out?
+    # check_rules(original_key_counter, key_layout, world, player)
 
     # Flip bk rules if more restrictive, to prevent placing a big key in a softlocking location
     for rule in key_logic.door_rules.values():
@@ -294,7 +320,7 @@ def create_exhaustive_placement_rules(key_layout, world, player):
                 else:
                     placement_self_lock_adjustment(rule, max_ctr, blocked_loc, key_counter, world, player)
                     rule.check_locations_w_bk = accessible_loc
-                    check_sm_restriction_needed(key_layout, max_ctr, rule, blocked_loc)
+                    # check_sm_restriction_needed(key_layout, max_ctr, rule, blocked_loc)
             else:
                 if big_key_progress(key_counter) and only_sm_doors(key_counter):
                     create_inclusive_rule(key_layout, max_ctr, code, key_counter, blocked_loc, accessible_loc, min_keys, world, player)
@@ -320,6 +346,7 @@ def placement_self_lock_adjustment(rule, max_ctr, blocked_loc, ctr, world, playe
             rule.needed_keys_w_bk -= 1
 
 
+# this rule is suspect - commented out usages for now
 def check_sm_restriction_needed(key_layout, max_ctr, rule, blocked):
     if rule.needed_keys_w_bk == key_layout.max_chests + len(max_ctr.key_only_locations):
         key_layout.key_logic.sm_restricted.update(blocked.difference(max_ctr.key_only_locations))
@@ -478,7 +505,7 @@ def create_inclusive_rule(key_layout, max_ctr, code, key_counter, blocked_loc, a
     else:
         placement_self_lock_adjustment(rule, max_ctr, blocked_loc, key_counter, world, player)
         rule.check_locations_w_bk = accessible_loc
-        check_sm_restriction_needed(key_layout, max_ctr, rule, blocked_loc)
+        # check_sm_restriction_needed(key_layout, max_ctr, rule, blocked_loc)
         key_logic.placement_rules.append(rule)
         adjust_locations_rules(key_logic, rule, accessible_loc, key_layout, key_counter, max_ctr)
 
@@ -538,6 +565,8 @@ def relative_empty_counter(odd_counter, key_counter):
         return False
     if len(set(odd_counter.free_locations).difference(key_counter.free_locations)) > 0:
         return False
+    if len(set(odd_counter.other_locations).difference(key_counter.other_locations)) > 0:
+        return False
     # important only
     if len(set(odd_counter.important_locations).difference(key_counter.important_locations)) > 0:
         return False
@@ -594,33 +623,50 @@ def unique_child_door_2(child, key_counter):
     return True
 
 
-def find_best_counter(door, odd_counter, key_counter, key_layout, world, player, skip_bk, empty_flag):  # try to waste as many keys as possible?
-    ignored_doors = {door, door.dest} if door is not None else {}
-    finished = False
-    opened_doors = dict(key_counter.open_doors)
-    bk_opened = key_counter.big_key_opened
-    # new_counter = key_counter
-    last_counter = key_counter
-    while not finished:
-        door_set = find_potential_open_doors(last_counter, ignored_doors, key_layout, skip_bk)
-        if door_set is None or len(door_set) == 0:
-            finished = True
-            continue
-        for new_door in door_set:
-            proposed_doors = {**opened_doors, **dict.fromkeys([new_door, new_door.dest])}
-            bk_open = bk_opened or new_door.bigKey
-            new_counter = find_counter(proposed_doors, bk_open, key_layout)
-            bk_open = new_counter.big_key_opened
-            # this means the new_door invalidates the door / leads to the same stuff
-            if not empty_flag and relative_empty_counter(odd_counter, new_counter):
-                ignored_doors.add(new_door)
-            elif empty_flag or key_wasted(new_door, door, last_counter, new_counter, key_layout, world, player):
-                last_counter = new_counter
-                opened_doors = proposed_doors
-                bk_opened = bk_open
-            else:
-                ignored_doors.add(new_door)
-    return last_counter
+# def find_best_counter(door, odd_counter, key_counter, key_layout, world, player, skip_bk, empty_flag):  # try to waste as many keys as possible?
+#     ignored_doors = {door, door.dest} if door is not None else {}
+#     finished = False
+#     opened_doors = dict(key_counter.open_doors)
+#     bk_opened = key_counter.big_key_opened
+#     # new_counter = key_counter
+#     last_counter = key_counter
+#     while not finished:
+#         door_set = find_potential_open_doors(last_counter, ignored_doors, key_layout, skip_bk)
+#         if door_set is None or len(door_set) == 0:
+#             finished = True
+#             continue
+#         for new_door in door_set:
+#             proposed_doors = {**opened_doors, **dict.fromkeys([new_door, new_door.dest])}
+#             bk_open = bk_opened or new_door.bigKey
+#             new_counter = find_counter(proposed_doors, bk_open, key_layout)
+#             bk_open = new_counter.big_key_opened
+#             # this means the new_door invalidates the door / leads to the same stuff
+#             if not empty_flag and relative_empty_counter(odd_counter, new_counter):
+#                 ignored_doors.add(new_door)
+#             elif empty_flag or key_wasted(new_door, door, last_counter, new_counter, key_layout, world, player):
+#                 last_counter = new_counter
+#                 opened_doors = proposed_doors
+#                 bk_opened = bk_open
+#             else:
+#                 ignored_doors.add(new_door)
+#     return last_counter
+
+
+def find_best_counter(door, key_layout, odd_counter, skip_bk, empty_flag):
+    best, best_ctr, locations = 0, None, 0
+    for code, counter in key_layout.key_counters.items():
+        if door not in counter.open_doors:
+            if best_ctr is None or counter.used_keys > best or (counter.used_keys == best and count_locations(counter) > locations):
+                if not skip_bk or not counter.big_key_opened:
+                    if empty_flag or not relative_empty_counter(odd_counter, counter):
+                        best = counter.used_keys
+                        best_ctr = counter
+                        locations = count_locations(counter)
+    return best_ctr
+
+
+def count_locations(ctr):
+    return len(ctr.free_locations) + len(ctr.key_only_locations) + len(ctr.other_locations) + len(ctr.important_locations)
 
 
 def find_worst_counter(door, odd_counter, key_counter, key_layout, skip_bk):  # try to waste as many keys as possible?
@@ -717,7 +763,7 @@ def calc_avail_keys(key_counter, world, player):
     return raw_avail - key_counter.used_keys
 
 
-def create_rule(key_counter, prev_counter, key_layout, world, player):
+def create_rule(key_counter, prev_counter, world, player):
     # prev_chest_keys = available_chest_small_keys(prev_counter, world)
     # prev_avail = prev_chest_keys + len(prev_counter.key_only_locations)
     chest_keys = available_chest_small_keys(key_counter, world, player)
@@ -736,6 +782,11 @@ def create_rule(key_counter, prev_counter, key_layout, world, player):
     return DoorRules(rule_num, is_valid)
 
 
+def create_worst_case_rule(rules, key_counter, world, player):
+    required_keys = key_counter.used_keys + 1  # this makes more sense, if key_counter has wasted all keys
+    rules.new_rules[KeyRuleType.WorstCase] = required_keys
+
+
 def check_for_self_lock_key(rule, door, parent_counter, key_layout, world, player):
     if world.accessibility[player] != 'locations':
         counter = find_inverted_counter(door, parent_counter, key_layout, world, player)
@@ -845,16 +896,16 @@ def big_key_drop_available(key_counter):
 def bk_restricted_rules(rule, door, odd_counter, empty_flag, key_counter, key_layout, world, player):
     if key_counter.big_key_opened:
         return
-    best_counter = find_best_counter(door, odd_counter, key_counter, key_layout, world, player, True, empty_flag)
-    bk_rule = create_rule(best_counter, key_counter, key_layout, world, player)
+    best_counter = find_best_counter(door, key_layout, odd_counter, True, empty_flag)
+    bk_rule = create_rule(best_counter, key_counter, world, player)
     if bk_rule.small_key_num >= rule.small_key_num:
         return
     door_open = find_next_counter(door, best_counter, key_layout)
     ignored_doors = dict_intersection(best_counter.child_doors, door_open.child_doors)
     dest_ignored = []
-    for door in ignored_doors.keys():
-        if door.dest not in ignored_doors:
-            dest_ignored.append(door.dest)
+    for d in ignored_doors.keys():
+        if d.dest not in ignored_doors:
+            dest_ignored.append(d.dest)
     ignored_doors = {**ignored_doors, **dict.fromkeys(dest_ignored)}
     post_counter = open_some_counter(door_open, key_layout, ignored_doors.keys())
     unique_loc = dict_difference(post_counter.free_locations, best_counter.free_locations)
@@ -862,8 +913,8 @@ def bk_restricted_rules(rule, door, odd_counter, empty_flag, key_counter, key_la
     if len(unique_loc) > 0:  # and bk_rule.is_valid
         rule.alternate_small_key = bk_rule.small_key_num
         rule.alternate_big_key_loc.update(unique_loc)
-    # elif not bk_rule.is_valid:
-    #     key_layout.key_logic.bk_restricted.update(unique_loc)
+        if not door.bigKey:
+            rule.new_rules[(KeyRuleType.Lock, key_layout.key_logic.bk_name)] = best_counter.used_keys + 1
 
 
 def find_worst_counter_wo_bk(small_key_num, accessible_set, door, odd_ctr, key_counter, key_layout):
@@ -935,6 +986,7 @@ def only_sm_doors(key_counter):
             return False
     return True
 
+
 # doesn't count dest doors
 def count_unique_small_doors(key_counter, proposal):
     cnt = 0
@@ -1197,7 +1249,7 @@ def check_rules_deep(original_counter, key_layout, world, player):
             elif not door.bigKey:
                 can_open = True
             if can_open:
-                can_progress = smalls_opened or not big_maybe_not_found
+                can_progress = (big_avail or not big_maybe_not_found) if door.bigKey else smalls_opened
                 next_counter = find_next_counter(door, counter, key_layout)
                 c_id = cid(next_counter, key_layout)
                 if c_id not in completed:
@@ -1381,6 +1433,7 @@ def cnt_avail_big_locations(ttl_locations, state, world, player):
 
 def create_key_counters(key_layout, world, player):
     key_counters = {}
+    key_layout.found_doors.clear()
     flat_proposal = key_layout.flat_prop
     state = ExplorationState(dungeon=key_layout.sector.name)
     if world.doorShuffle[player] == 'vanilla':
@@ -1403,6 +1456,9 @@ def create_key_counters(key_layout, world, player):
     while len(queue) > 0:
         next_key_counter, parent_state = queue.popleft()
         for door in next_key_counter.child_doors:
+            key_layout.found_doors.add(door)
+            if door.dest in flat_proposal and door.type != DoorType.SpiralStairs:
+                key_layout.found_doors.add(door.dest)
             child_state = parent_state.copy()
             if door.bigKey or door.name in special_big_key_doors:
                 key_layout.key_logic.bk_doors.add(door)
@@ -1520,11 +1576,11 @@ def find_counter_hint(opened_doors, bk_hint, key_layout):
 
 
 def find_max_counter(key_layout):
-    max_counter = find_counter_hint(dict.fromkeys(key_layout.flat_prop), False, key_layout)
+    max_counter = find_counter_hint(dict.fromkeys(key_layout.found_doors), False, key_layout)
     if max_counter is None:
         raise Exception("Max Counter is none - something is amiss")
     if len(max_counter.child_doors) > 0:
-        max_counter = find_counter_hint(dict.fromkeys(key_layout.flat_prop), True, key_layout)
+        max_counter = find_counter_hint(dict.fromkeys(key_layout.found_doors), True, key_layout)
     return max_counter
 
 
diff --git a/Main.py b/Main.py
index 9600fa8f..74d4c77e 100644
--- a/Main.py
+++ b/Main.py
@@ -448,6 +448,7 @@ def copy_world(world):
         # these need to be modified properly by set_rules
         new_location.access_rule = lambda state: True
         new_location.item_rule = lambda state: True
+        new_location.forced_item = location.forced_item
 
     # copy remaining itempool. No item in itempool should have an assigned location
     for item in world.itempool:
diff --git a/Rules.py b/Rules.py
index 3762b9bb..b30780ed 100644
--- a/Rules.py
+++ b/Rules.py
@@ -3,7 +3,7 @@ import logging
 from collections import deque
 
 import OverworldGlitchRules
-from BaseClasses import CollectionState, RegionType, DoorType, Entrance, CrystalBarrier
+from BaseClasses import CollectionState, RegionType, DoorType, Entrance, CrystalBarrier, KeyRuleType
 from RoomData import DoorKind
 from OverworldGlitchRules import overworld_glitches_rules
 
@@ -1939,14 +1939,10 @@ bunny_impassible_doors = {
 def add_key_logic_rules(world, player):
     key_logic = world.key_logic[player]
     for d_name, d_logic in key_logic.items():
-        for door_name, keys in d_logic.door_rules.items():
-            spot = world.get_entrance(door_name, player)
-            if not world.retro[player] or world.mode[player] != 'standard' or not retro_in_hc(spot):
-                rule = create_advanced_key_rule(d_logic, player, keys)
-                if keys.opposite:
-                    rule = or_rule(rule, create_advanced_key_rule(d_logic, player, keys.opposite))
-                add_rule(spot, rule)
-
+        for door_name, rule in d_logic.door_rules.items():
+            add_rule(world.get_entrance(door_name, player), eval_small_key_door(door_name, d_name, player))
+            if rule.allow_small:
+                set_always_allow(rule.small_location, allow_self_locking_small(d_logic, player))
         for location in d_logic.bk_restricted:
             if not location.forced_item:
                 forbid_item(location, d_logic.bk_name, player)
@@ -1954,8 +1950,9 @@ def add_key_logic_rules(world, player):
             forbid_item(location, d_logic.small_key_name, player)
         for door in d_logic.bk_doors:
             add_rule(world.get_entrance(door.name, player), create_rule(d_logic.bk_name, player))
-        for chest in d_logic.bk_chests:
-            add_rule(world.get_location(chest.name, player), create_rule(d_logic.bk_name, player))
+        if len(d_logic.bk_doors) > 0 or len(d_logic.bk_chests) > 1:
+            for chest in d_logic.bk_chests:
+                add_rule(world.get_location(chest.name, player), create_rule(d_logic.bk_name, player))
     if world.retro[player]:
         for d_name, layout in world.key_layout[player].items():
             for door in layout.flat_prop:
@@ -1963,6 +1960,39 @@ def add_key_logic_rules(world, player):
                     add_rule(door.entrance, create_key_rule('Small Key (Universal)', player, 1))
 
 
+def allow_self_locking_small(logic, player):
+    return lambda state, item: item.player == player and logic.small_key_name == item.name
+
+
+def eval_small_key_door_main(state, door_name, dungeon, player):
+    if state.is_door_open(door_name, player):
+        return True
+    key_logic = state.world.key_logic[player][dungeon]
+    door_rule = key_logic.door_rules[door_name]
+    door_openable = False
+    for ruleType, number in door_rule.new_rules.items():
+        if door_openable:
+            return True
+        if ruleType == KeyRuleType.WorstCase:
+            door_openable |= state.has_sm_key(key_logic.small_key_name, player, number)
+        elif ruleType == KeyRuleType.AllowSmall:
+            if door_rule.small_location.item and door_rule.small_location.item.name == key_logic.small_key_name:
+                return True  # always okay if allow small is on
+        elif isinstance(ruleType, tuple):
+            lock, lock_item = ruleType
+            # this doesn't track logical locks yet, i.e. hammer locks the item and hammer is there, but the item isn't
+            for loc in door_rule.alternate_big_key_loc:
+                spot = state.world.get_location(loc, player)
+                if spot.item and spot.item.name == lock_item:
+                    door_openable |= state.has_sm_key(key_logic.small_key_name, player, number)
+                    break
+    return door_openable
+
+
+def eval_small_key_door(door_name, dungeon, player):
+    return lambda state: eval_small_key_door_main(state, door_name, dungeon, player)
+
+
 def retro_in_hc(spot):
     return spot.parent_region.dungeon.name == 'Hyrule Castle' if spot.parent_region.dungeon else False
 
diff --git a/Utils.py b/Utils.py
index cf3db3fb..b8cf5497 100644
--- a/Utils.py
+++ b/Utils.py
@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 import os
 import re
+import operator as op
 import subprocess
 import sys
 import xml.etree.ElementTree as ET
 from collections import defaultdict
+from functools import reduce
 
 
 def int16_as_bytes(value):
@@ -116,6 +118,28 @@ def make_new_base2current(old_rom='Zelda no Densetsu - Kamigami no Triforce (Jap
     return "New Rom Hash: " + basemd5.hexdigest()
 
 
+def kth_combination(k, l, r):
+    if r == 0:
+        return []
+    elif len(l) == r:
+        return l
+    else:
+        i = ncr(len(l)-1, r-1)
+        if k < i:
+            return l[0:1] + kth_combination(k, l[1:], r-1)
+        else:
+            return kth_combination(k-i, l[1:], r)
+
+
+def ncr(n, r):
+    if r == 0:
+        return 1
+    r = min(r, n-r)
+    numerator = reduce(op.mul, range(n, n-r, -1), 1)
+    denominator = reduce(op.mul, range(1, r+1), 1)
+    return numerator / denominator
+
+
 entrance_offsets = {
     'Sanctuary': 0x2,
     'HC West': 0x3,