Fix chained selection xor and subtraction operations

Something that's reasonable to expect to see in Lua files is something
like:

    local sel4 = sel1 - sel2 - sel3

or more generally, producing a selection from subtraction that will then
be used in subsequent selection math.

I discovered this wasn't actually working correctly, and that it also
applied to the xor operation. The reason behind this is that
l_selection_sub and l_selection_xor create a new selection from nothing,
which by default has "lower" bounds of COLNO, ROWNO and "upper" bounds
of 0,0. Iterating across the intersecting rectangle of both selections
does not reliably set the bounds of the resulting selection properly,
since the first selection_setpoint with a value of 0 will cause the
selection's bounds_dirty flag to be set, at which point they will cease
to change as more points are added.

Then this selection with its incorrect boundaries is pushed back onto
the Lua stack, and becomes the first operand of the next subtraction
(i.e. selr in the first l_selection_sub becomes sela in the second
l_selection_sub). Depending on how broken the bounding box is, results
may vary, but if the bounding box is still (COLNO,ROWNO,0,0), the
resulting selection will have no points selected at all.

This fixes this problem by forcibly recalculating the bounds of the
result selection, so any subsequent operations on it will be valid.
This commit is contained in:
copperwater
2023-05-20 18:35:49 -04:00
committed by PatR
parent 6e3ccaa8a5
commit c6fa9c3099
3 changed files with 8 additions and 2 deletions

View File

@@ -2677,6 +2677,7 @@ extern void selection_free(struct selectionvar *, boolean);
extern void selection_clear(struct selectionvar *, int);
extern struct selectionvar *selection_clone(struct selectionvar *);
extern void selection_getbounds(struct selectionvar *, NhRect *);
extern void selection_recalc_bounds(struct selectionvar *);
extern void set_selection_floodfillchk(int(*)(coordxy, coordxy));
extern void selection_floodfill(struct selectionvar *, coordxy, coordxy,
boolean);

View File

@@ -337,6 +337,9 @@ l_selection_xor(lua_State *L)
int val = selection_getpoint(x, y, sela) ^ selection_getpoint(x, y, selb);
selection_setpoint(x, y, selr, val);
}
/* this may have created a smaller or irregular selection with bounds_dirty
* set to true - update its boundaries */
selection_recalc_bounds(selr);
lua_remove(L, 1);
lua_remove(L, 1);
@@ -363,6 +366,9 @@ l_selection_sub(lua_State *L)
int val = (a_pt ^ b_pt) & a_pt;
selection_setpoint(x, y, selr, val);
}
/* this may have created a smaller or irregular selection with bounds_dirty
* set to true - update its boundaries */
selection_recalc_bounds(selr);
lua_remove(L, 1);
lua_remove(L, 1);

View File

@@ -124,7 +124,6 @@ static const char *get_mkroom_name(int);
static int get_table_roomtype_opt(lua_State *, const char *, int);
static int get_table_traptype_opt(lua_State *, const char *, int);
static int get_traptype_byname(const char *);
static void selection_recalc_bounds(struct selectionvar *);
static lua_Integer get_table_intarray_entry(lua_State *, int, int);
static struct sp_coder *sp_level_coder_init(void);
@@ -4562,7 +4561,7 @@ selection_getbounds(struct selectionvar *sel, NhRect *b)
}
/* recalc the boundary of selection, if necessary */
static void
void
selection_recalc_bounds(struct selectionvar *sel)
{
coordxy x, y;