sort of/kind of support PANICTRACE on VMS (trunk only)

I don't think this is useful enough to recommend ordinary users
enable it, but it's close enough to being useful that I don't want
to leave it to become subject to bit rot like umpteen other unfinished
patches.  Anyone running in wizard mode who has a panic already gets
pushed into the debugger on VMS, although it doesn't work for what might
be considered the most important configuration (a secure playground, as
opposed to the wide-open one I've always been content to leave mine at).

include/extern.h

@@ -2486,6 +2486,9 @@ E boolean NDECL(authorize_wizard_mode);
 
 E void NDECL(vms_abort);
 E void FDECL(vms_exit, (int));
+#ifdef PANICTRACE
+E void FDECL(vms_traceback, (int));
+#endif
 
 /* ### vmstty.c ### */
 

include/vmsconf.h

@@ -73,6 +73,20 @@
  */
 /* #define SECURE */
 
+/*
+ * If you use SECURE you'll need to link /noTraceback, in which case
+ * there's no point trying to get extra PANICTRACE info and this might
+ * as well be commented out.  When enabled, the sysconf file controls
+ * how to handle it (note that we're hijacking the Unix GDB setting):
+PANICTRACE_GDB=0  #behave as if PANICTRACE was disabled
+PANICTRACE_GDB=1  #at conclusion of panic, show a call traceback and exit
+PANICTRACE_GDB=2  #at conclusion of panic, show a call traceback and then
+ *                # remain in the debugger for more interactive debugging
+ *                # (not as useful as it might sound since we're normally
+ *                # linked /noDebug so there's no symbol table accessible)
+ */
+/* #define PANICTRACE */
+
 /*
  * Put the readonly data files into a single container rather than into
  * separate files in the playground directory.

src/end.c

@@ -160,7 +160,8 @@ boolean set;
 # endif	/* NO_SIGNAL */
 
 static void
-NH_abort(){
+NH_abort()
+{
 	int gdb_prio = SYSOPT_PANICTRACE_GDB;
 	int glibc_prio = SYSOPT_PANICTRACE_GLIBC;
 	static boolean aborting = FALSE;
@@ -168,6 +169,7 @@ NH_abort(){
 	if(aborting) return;
 	aborting = TRUE;
 
+# ifndef VMS
 	if(gdb_prio == glibc_prio && gdb_prio > 0) gdb_prio++;
 
 	if(gdb_prio > glibc_prio){
@@ -176,6 +178,15 @@ NH_abort(){
 		NH_panictrace_glibc() || (gdb_prio && NH_panictrace_gdb());
 	}
 
+# else /* VMS */
+	/* overload otherwise unused priority for debug mode: 1 = show
+	   traceback and exit; 2 = show traceback and stay in debugger */
+     /* if (wizard && gdb_prio == 1) gdb_prio = 2; */
+	vms_traceback(gdb_prio);
+	(void)glibc_prio;	/* half-hearted attempt at lint suppression */
+
+# endif /* ?VMS */
+
 # ifndef NO_SIGNAL
 	panictrace_setsignals(FALSE);
 # endif
@@ -183,7 +194,8 @@ NH_abort(){
 }
 
 static boolean
-NH_panictrace_glibc(){
+NH_panictrace_glibc()
+{
 # ifdef PANICTRACE_GLIBC
 	void *bt[20];
 	size_t count;
@@ -218,7 +230,8 @@ NH_panictrace_glibc(){
 # endif  /* PANICTRACE_GDB */
 
 static boolean
-NH_panictrace_gdb(){
+NH_panictrace_gdb()
+{
 # ifdef PANICTRACE_GDB
 	/* A (more) generic method to get a stack trace - invoke
 	 * gdb on ourself. */

sys/vms/Makefile.src

@@ -1,9 +1,9 @@
 #	NetHack Makefile (VMS) - for building nethack itself.
-#	SCCS Id: @(#)Makefile.src	3.5	2008/01/30
+#	NetHack 3.5	Makefile.src	$Date$	$Revision$
 
 #  Copy this file to [.src]Makefile. and then edit it as needed.
 #  The default configuration is for building with DEC C (aka Compaq C).
-#  If you changed CC or CFLAGS, make similar changes in [.util]Makefile.
+#  If you change CC or CFLAGS, make similar changes in [.util]Makefile.
 #
 #  Note:  modifying this Makefile will cause crtl.opt to be rebuilt,
 #	which will trigger an update of makedefs, which will in turn
@@ -315,7 +315,7 @@ $(HACK_H) : $(INC)hack.h $(CONFIG_H) $(INC)align.h \
 vmsmain.obj :	$(VMS)vmsmain.c $(HACK_H) $(INC)dlb.h
 vmstty.obj :	$(VMS)vmstty.c $(HACK_H) $(INC)wintty.h $(INC)tcap.h
 vmsunix.obj :	$(VMS)vmsunix.c $(HACK_H)
-vmsmisc.obj :	$(VMS)vmsmisc.c $(VMS)oldcrtl.c
+vmsmisc.obj :	$(VMS)vmsmisc.c $(VMS)oldcrtl.c $(CONFIG_H)
 vmsfiles.obj :	$(VMS)vmsfiles.c $(CONFIG_H)
 vmsmail.obj :	$(VMS)vmsmail.c $(CONFIG_H) $(INC)mail.h \
 		$(INC)wintype.h $(INC)winprocs.h

sys/vms/Makefile.utl

@@ -1,5 +1,5 @@
 #	NetHack Makefile (VMS) - for utility programs.
-#	SCCS Id: @(#)Makefile.utl	3.5	2007/10/27
+#	NetHack 3.5	Makefile.utl	$Date$	$Revision$
 
 #  Copy this file to [.util]Makefile. and then edit it as needed.
 #  The default configuration is for building with DEC C (aka Compaq C).
@@ -341,7 +341,7 @@ $(CONFIG_H) : $(INC)config.h
       @ $(CD) $(UTL)
 
 # VMS specific dependencies
-$(SRC)vmsmisc.obj :	$(VMS)vmsmisc.c
+$(SRC)vmsmisc.obj :	$(VMS)vmsmisc.c $(CONFIG_H)
 	$(CD) $(SRC)
 	$(MAKE)$(MAKEFLAGS) vmsmisc.obj
       @ $(CD) $(UTL)

sys/vms/vmsmisc.c

@@ -1,14 +1,16 @@
 /* NetHack 3.5	vmsmisc.c	$Date$  $Revision$ */
-/*	SCCS Id: @(#)vmsmisc.c	3.5	1996/03/02	*/
 /* NetHack may be freely redistributed.  See license for details. */
 
+#include "config.h"
+#undef exit
 #include <ssdef.h>
 #include <stsdef.h>
 
-void vms_exit( /*_ int _*/ );
+void FDECL(vms_exit, (int));
-void vms_abort( /*_ void _*/ );
+void NDECL(vms_abort);
+
+extern int FDECL(vms_define, (const char *,const char *,int));
 
-extern void exit( /*_ int _*/ );
 extern void lib$signal( /*_ unsigned long,... _*/ );
 
 void
@@ -24,6 +26,61 @@ vms_abort()
     lib$signal(SS$_DEBUG);
 }
 
+#ifdef PANICTRACE
+void
+vms_traceback(how)
+int how;	/* 1: exit after traceback; 2: stay in debugger */
+{
+    /* signal handler expects first byte to hold length of the rest */
+    char dbgcmd[1+255];
+
+    dbgcmd[0] = dbgcmd[1] = '\0';
+    if (how == 2) {
+	/* limit output to 18 stack frames to avoid longer output causing
+	   nethack's panic prolog from scrolling off conventional sized
+	   screen; perhaps we should adapt to termcap LI here... */
+	(void)strcpy(dbgcmd, "#set Module/Calls; show Calls 18");
+    } else if (how == 1) {
+	/*
+	 * Suppress most of debugger's initial feedback to avoid scaring users.
+	 */
+	/* start up with output going to /dev/null instead of stdout */
+	(void)vms_define("DBG$OUTPUT", "_NL:", 0);
+	/* bypass any debugger initialization file the user might have */
+	(void)vms_define("DBG$INIT", "_NL:", 0);
+	/* force tty interface by suppressing DECwindows/Motif interface */
+	(void)vms_define("DBG$DECW$DISPLAY", " ", 0);
+	/* once started, send output to log file on stdout */
+	(void)strcpy(dbgcmd, "#set Log SYS$OUTPUT:; set output Log,noTerminal");
+	/* FIXME: the trailing exit command here is actually being ignored,
+	   leaving us at the DBG> prompt contrary to our intent... */
+	(void)strcat(dbgcmd, "; set Module/Calls; show Calls 18; exit");
+    }
+    
+    if (dbgcmd[1]) {
+	/* plug in command's length; debugger's signal handler expects ASCIC
+	   counted string rather than C-style ASCIZ 0-terminated string */
+	dbgcmd[0] = (char)strlen(&dbgcmd[1]);
+	/*
+	 * This won't work if we've been linked /noTraceback, and
+	 * we have to link /noTraceback if nethack.exe is going
+	 * to be installed with privileges, so this is of dubious
+	 * value for a SECURE multi-user playground installation.
+	 *
+	 * TODO: What's worse, we need to add a condition handler
+	 * to trap the resulting "improperly handled condition"
+	 * and the annoying and/or frightening (and in this case,
+	 * useless) register dump given when the debugger can't be
+	 * activated for a noTraceback executable.
+	 */
+	(void)lib$signal(SS$_DEBUG, 1, dbgcmd);
+    }
+
+    vms_exit(2);	/* don't return to caller */
+    /* NOT REACHED */
+}
+#endif
+
 #ifdef VERYOLD_VMS
 #include "oldcrtl.c"
 #endif