From c6fa9c30999e8457e717a15cb316798cf241592d Mon Sep 17 00:00:00 2001 From: copperwater Date: Sat, 20 May 2023 18:35:49 -0400 Subject: [PATCH] Fix chained selection xor and subtraction operations Something that's reasonable to expect to see in Lua files is something like: local sel4 = sel1 - sel2 - sel3 or more generally, producing a selection from subtraction that will then be used in subsequent selection math. I discovered this wasn't actually working correctly, and that it also applied to the xor operation. The reason behind this is that l_selection_sub and l_selection_xor create a new selection from nothing, which by default has "lower" bounds of COLNO, ROWNO and "upper" bounds of 0,0. Iterating across the intersecting rectangle of both selections does not reliably set the bounds of the resulting selection properly, since the first selection_setpoint with a value of 0 will cause the selection's bounds_dirty flag to be set, at which point they will cease to change as more points are added. Then this selection with its incorrect boundaries is pushed back onto the Lua stack, and becomes the first operand of the next subtraction (i.e. selr in the first l_selection_sub becomes sela in the second l_selection_sub). Depending on how broken the bounding box is, results may vary, but if the bounding box is still (COLNO,ROWNO,0,0), the resulting selection will have no points selected at all. This fixes this problem by forcibly recalculating the bounds of the result selection, so any subsequent operations on it will be valid. --- include/extern.h | 1 + src/nhlsel.c | 6 ++++++ src/sp_lev.c | 3 +-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/include/extern.h b/include/extern.h index 0f62485d6..375acdf22 100644 --- a/include/extern.h +++ b/include/extern.h @@ -2677,6 +2677,7 @@ extern void selection_free(struct selectionvar *, boolean); extern void selection_clear(struct selectionvar *, int); extern struct selectionvar *selection_clone(struct selectionvar *); extern void selection_getbounds(struct selectionvar *, NhRect *); +extern void selection_recalc_bounds(struct selectionvar *); extern void set_selection_floodfillchk(int(*)(coordxy, coordxy)); extern void selection_floodfill(struct selectionvar *, coordxy, coordxy, boolean); diff --git a/src/nhlsel.c b/src/nhlsel.c index 52850d8f6..8694f7147 100644 --- a/src/nhlsel.c +++ b/src/nhlsel.c @@ -337,6 +337,9 @@ l_selection_xor(lua_State *L) int val = selection_getpoint(x, y, sela) ^ selection_getpoint(x, y, selb); selection_setpoint(x, y, selr, val); } + /* this may have created a smaller or irregular selection with bounds_dirty + * set to true - update its boundaries */ + selection_recalc_bounds(selr); lua_remove(L, 1); lua_remove(L, 1); @@ -363,6 +366,9 @@ l_selection_sub(lua_State *L) int val = (a_pt ^ b_pt) & a_pt; selection_setpoint(x, y, selr, val); } + /* this may have created a smaller or irregular selection with bounds_dirty + * set to true - update its boundaries */ + selection_recalc_bounds(selr); lua_remove(L, 1); lua_remove(L, 1); diff --git a/src/sp_lev.c b/src/sp_lev.c index db20cb745..637fd3c65 100644 --- a/src/sp_lev.c +++ b/src/sp_lev.c @@ -124,7 +124,6 @@ static const char *get_mkroom_name(int); static int get_table_roomtype_opt(lua_State *, const char *, int); static int get_table_traptype_opt(lua_State *, const char *, int); static int get_traptype_byname(const char *); -static void selection_recalc_bounds(struct selectionvar *); static lua_Integer get_table_intarray_entry(lua_State *, int, int); static struct sp_coder *sp_level_coder_init(void); @@ -4562,7 +4561,7 @@ selection_getbounds(struct selectionvar *sel, NhRect *b) } /* recalc the boundary of selection, if necessary */ -static void +void selection_recalc_bounds(struct selectionvar *sel) { coordxy x, y;