PANICTRACE on VMS (trunk only)

The preliminary implementation of PANICTRACE on VMS had a "Fixme"
that this fixes, and a "TODO" that this makes moot, but the main reason
for this patch is that vmsmisc.c had been changed to call vms_define(),
which resides in vmsunix.c.  Since vmsmisc.obj is linked into progarms
in util/ and vmsunix.obj isn't, enabling PANICTRACE caused linking
problems for those.  This moves the code that wants to call vms_define()
into vmsunix.c (despite the fact that it's not even vaguely related to
Unix emulation), so that it only matters to nethack and doesn't impact
the utility programs anymore.

     This uses a VMS facility called LIB$INITIALIZE to call code before
main() starts.  It's rather messy--at least when written in something
other than assembler or Bliss--and shouldn't be needed for nethack,
but I couldn't figure out how to trap the condition signalled by
lib$signal(SS$_DEBUG) when the debugger isn't available to do so, so I
needed a way to make issuing that signal be conditional upon debugger
availability.  One of the arguments passed to LIB$INITIALIZE-invoked
routines contains information that makes if feasible to deduce whether
the debugger is available.

     Even when PANICTRACE is disabled, that's useful for handling abort
due to panic while in running in wizard mode.

sys/vms/vmsmisc.c

@@ -6,81 +6,40 @@
 #include <ssdef.h>
 #include <stsdef.h>
 
+int debuggable = 0;	/* 1 if we can debug or show a call trace */
+
 void FDECL(vms_exit, (int));
 void NDECL(vms_abort);
 
-extern int FDECL(vms_define, (const char *,const char *,int));
-
-extern void lib$signal( /*_ unsigned long,... _*/ );
+/* first arg should be unsigned long but <lib$routines.h> has unsigned int */
+extern void VDECL(lib$signal, (unsigned,...));
 
+/* terminate, converting Unix-style exit code into VMS status code */
 void
 vms_exit(status)
 int status;
 {
+    /* convert non-zero to failure, zero to success */
     exit(status ? (SS$_ABORT | STS$M_INHIB_MSG) : SS$_NORMAL);
+    /* NOT REACHED */
 }
 
+/* put the user into the debugger; used for abort() when in wizard mode */
 void
 vms_abort()
 {
-    lib$signal(SS$_DEBUG);
-}
+    if (debuggable)
+	lib$signal(SS$_DEBUG);
 
-#ifdef PANICTRACE
-void
-vms_traceback(how)
-int how;	/* 1: exit after traceback; 2: stay in debugger */
-{
-    /* signal handler expects first byte to hold length of the rest */
-    char dbgcmd[1+255];
-
-    dbgcmd[0] = dbgcmd[1] = '\0';
-    if (how == 2) {
-	/* limit output to 18 stack frames to avoid longer output causing
-	   nethack's panic prolog from scrolling off conventional sized
-	   screen; perhaps we should adapt to termcap LI here... */
-	(void)strcpy(dbgcmd, "#set Module/Calls; show Calls 18");
-    } else if (how == 1) {
-	/*
-	 * Suppress most of debugger's initial feedback to avoid scaring users.
-	 */
-	/* start up with output going to /dev/null instead of stdout */
-	(void)vms_define("DBG$OUTPUT", "_NL:", 0);
-	/* bypass any debugger initialization file the user might have */
-	(void)vms_define("DBG$INIT", "_NL:", 0);
-	/* force tty interface by suppressing DECwindows/Motif interface */
-	(void)vms_define("DBG$DECW$DISPLAY", " ", 0);
-	/* once started, send output to log file on stdout */
-	(void)strcpy(dbgcmd, "#set Log SYS$OUTPUT:; set output Log,noTerminal");
-	/* FIXME: the trailing exit command here is actually being ignored,
-	   leaving us at the DBG> prompt contrary to our intent... */
-	(void)strcat(dbgcmd, "; set Module/Calls; show Calls 18; exit");
-    }
-    
-    if (dbgcmd[1]) {
-	/* plug in command's length; debugger's signal handler expects ASCIC
-	   counted string rather than C-style ASCIZ 0-terminated string */
-	dbgcmd[0] = (char)strlen(&dbgcmd[1]);
-	/*
-	 * This won't work if we've been linked /noTraceback, and
-	 * we have to link /noTraceback if nethack.exe is going
-	 * to be installed with privileges, so this is of dubious
-	 * value for a SECURE multi-user playground installation.
-	 *
-	 * TODO: What's worse, we need to add a condition handler
-	 * to trap the resulting "improperly handled condition"
-	 * and the annoying and/or frightening (and in this case,
-	 * useless) register dump given when the debugger can't be
-	 * activated for a noTraceback executable.
-	 */
-	(void)lib$signal(SS$_DEBUG, 1, dbgcmd);
-    }
-
-    vms_exit(2);	/* don't return to caller */
+    /* we'll get here if the debugger isn't available, or if the user
+       uses GO to resume execution instead of EXIT to quit */
+    vms_exit(2);	/* don't return to caller (2==arbitrary non-zero) */
     /* NOT REACHED */
 }
-#endif
 
+    /*
+     * Caveat: the VERYOLD_VMS configuration hasn't been tested in many years.
+     */
 #ifdef VERYOLD_VMS
 #include "oldcrtl.c"
 #endif