Make mstate flag checks more mutually consistent

Various places checking for whether a monster was on the map based on
mstate flags were inconsistent about which ones they checked (and then
place_monster() was additionally inconsistent with all of them about
which bits were cleared when placing a monster onto the map).  I think
some places were also more convoluted than is now necessary because they
date back to mstate being an alias for mspare1, which it shared with
migflags before those became two separate dedicated fields (MSTATE_MASK
also dates back to this and is no longer used, so I removed it).

I tried to go through all the MON_foo mstate bits, understand when/why
they are set, and make the various functions I noticed more consistent
(with each other, and with my understanding of how the bits work) about
how they are treated.  I don't know for a fact that I understood
everything right -- some diagnostic bits that aren't used for much of
anything, like MON_OBLITERATE, had me mystified until I read the 5ee78c5
commit message -- but this patch hasn't caused any new problems (sanity
check or otherwise) with the fuzzer in my testing so far.  All the same,
it could probably use review by someone who has a good sense of what the
mstate bits mean.

include/monst.h

@@ -64,7 +64,6 @@ enum m_ap_types {
 #define MON_ENDGAME_FREE 0x20
 #define MON_ENDGAME_MIGR 0x40
 #define MON_OBLITERATE   0x80
-#define MSTATE_MASK      0xFF
 
 #define M_AP_TYPMASK  0x7
 #define M_AP_F_DKNOWN 0x8
@@ -250,7 +249,7 @@ struct monst {
 
 #define mon_perma_blind(mon) (!mon->mcansee && !mon->mblinded)
 
-#define mon_offmap(mon) (((mon)->mstate & (MON_DETACH|MON_MIGRATING|MON_LIMBO|MON_OFFMAP)) != 0)
+#define mon_offmap(mon) ((mon)->mstate != MON_FLOOR)
 
 /* Get the maximum difficulty monsters that can currently be generated,
    given the current level difficulty and the hero's level. */

src/cmd.c

@@ -1057,7 +1057,7 @@ makemap_unmakemon(struct monst *mtmp, boolean migratory)
            monsters won't get out of sync; it is not on the map but
            mongone() -> m_detach() -> mon_leaving_level() copes with that */
         mtmp->mstate |= MON_OFFMAP;
-        mtmp->mstate &= ~(MON_MIGRATING | MON_LIMBO);
+        mtmp->mstate &= ~(MON_MIGRATING | MON_LIMBO | MON_ENDGAME_MIGR);
         mtmp->nmon = fmon;
         fmon = mtmp;
     }

src/mhitm.c

@@ -1048,8 +1048,8 @@ mdamagem(
 
     if (mhitm_knockback(magr, mdef, mattk, &mhm.hitflags,
                         (MON_WEP(magr) != 0))
-        && ((mhm.hitflags & (M_ATTK_DEF_DIED|M_ATTK_HIT)) != 0
-            || (mdef->mstate & (MON_DETACH|MON_MIGRATING|MON_LIMBO)) != 0))
+        && ((mhm.hitflags & (M_ATTK_DEF_DIED | M_ATTK_HIT)) != 0
+            || mon_offmap(mdef)))
         return mhm.hitflags;
 
     if (mhm.done)

src/mon.c

@@ -237,9 +237,9 @@ mon_sanity_check(void)
         } else if (mtmp->wormno) {
             sanity_check_worm(mtmp);
 
-        /* TODO: figure out which other bits shouldn't be set for 'fmon' */
-        } else if ((mtmp->mstate & (MON_DETACH | MON_MIGRATING | MON_LIMBO))
-                   != 0) {
+        /* some temp mstate bits can be expected for a mon on fmon, as part of
+           removing it, but DEADMONSTER check above should skip those. */
+        } else if (mon_offmap(mtmp)) {
             impossible("floor mon (%s) with mstate set to 0x%08lx",
                        fmt_ptr((genericptr_t) mtmp), mtmp->mstate);
         }
@@ -267,10 +267,9 @@ mon_sanity_check(void)
     for (mtmp = gm.migrating_mons; mtmp; mtmp = mtmp->nmon) {
         sanity_check_single_mon(mtmp, FALSE, "migr");
 
-        /* TODO: figure out which other bits could or shouldn't be set
-         * when migrating */
-        if ((mtmp->mstate & (MON_DETACH)) != 0
-            || (mtmp->mstate & (MON_MIGRATING | MON_LIMBO)) == 0)
+        if ((mtmp->mstate
+             & ~(MON_MIGRATING | MON_LIMBO | MON_ENDGAME_MIGR)) != 0L
+            || !(mtmp->mstate & MON_MIGRATING))
             impossible("migrating mon (%s) with mstate set to 0x%08lx",
                        fmt_ptr((genericptr_t) mtmp), mtmp->mstate);
     }

src/steed.c

@@ -873,8 +873,7 @@ place_monster(struct monst* mon, coordxy x, coordxy y)
     }
     mon->mx = x, mon->my = y;
     gl.level.monsters[x][y] = mon;
-    mon->mstate &= ~(MON_OFFMAP | MON_MIGRATING | MON_LIMBO | MON_BUBBLEMOVE
-                     | MON_ENDGAME_FREE | MON_ENDGAME_MIGR);
+    mon->mstate = MON_FLOOR;
 }
 
 /*steed.c*/