Finally remove AST vmalloc(3) (re: f9364b1, 3567220)

clang's -fsanitize=undefined option is flagging undefined behaviour
in AST's vmalloc memory manager. Vmalloc is old, obsolete and
unmaintained. At this point, it has no real added value, even for
the ksh memory leak tests (see d845675). It's well past time to
get rid completely and just use the local operating system's
time-tested and (usually) efficient malloc(3) implementation. This
commit does all that in one fell swoop.

Main changes follow (related tweaks are too many to list):

src/lib/libast/cdt/dtnew.c,
src/lib/libast/misc/fastfind.c,
src/lib/libast/misc/magic.c,
src/lib/libast/port/mc.c,
src/lib/libdll/dllscan.c,
src/cmd/builtin/pty.c:
- Change all vmalloc-specific calls to standard calls. This mostly
  involves removing vmopen() and vmclose() calls and porting
  vmnewof() and vmfree() calls to standard calloc(3) and free(3).

src/cmd/ksh93/**:
- Remove all the !_std_malloc code.

src/lib/libast/man/vmalloc.3,
src/lib/libast/include/vmalloc.h,
src/lib/libast/disc/memfatal.c,
src/lib/libcmd/vmstate.c,
src/lib/libast/vmalloc/*:
- Removed.

This commit is partially based on prior work done by ksh2020 in:
att@489c672d

src/cmd/builtin/Mamfile

@@ -31,7 +31,6 @@ make install virtual
 					exec -	-I%{INSTALLROOT}/include %{mam_libast} %{mam_libcmd} : run %{<}
 				done
 				prev %{INCLUDE_AST}/ast_time.h
-				prev %{INCLUDE_AST}/vmalloc.h
 				prev %{INCLUDE_AST}/regex.h
 				prev %{INCLUDE_AST}/proc.h
 				prev %{INCLUDE_AST}/error.h

src/cmd/builtin/pty.c

@@ -18,7 +18,7 @@
 ***********************************************************************/
 
 static const char usage[] =
-"[-?\n@(#)pty (AT&T Research) 2013-05-22\n]"
+"[-?\n@(#)pty (ksh 93u+m) 2024-07-27\n]"
 "[-author?Glenn Fowler <gsf@research.att.com>]"
 "[-author?David Korn <dgk@research.att.com>]"
 "[-copyright?Copyright (c) 2001-2013 AT&T Intellectual Property]"
@@ -109,7 +109,6 @@ static const char usage[] =
 #include <proc.h>
 #include <ctype.h>
 #include <regex.h>
-#include <vmalloc.h>
 #include <ast_time.h>
 #include <sys/wait.h>
 #include <sys/stat.h>
@@ -497,7 +496,6 @@ match(char* pattern, char* text, int must)
 
 typedef struct Master_s
 {
-	Vmalloc_t*	vm;		/* allocation region			*/
 	char*		ignore;		/* ignore master lines matching this re	*/
 	char*		peek;		/* peek buffer pointer			*/
 	char*		cur;		/* current line				*/
@@ -525,7 +523,6 @@ masterline(Sfio_t* mp, Sfio_t* lp, char* prompt, int must, int timeout, Master_t
 	char*		s;
 	char*		t;
 	ssize_t		n;
-	ssize_t		a;
 	size_t		promptlen = 0;
 	ptrdiff_t	d;
 	char		promptbuf[64];
@@ -652,11 +649,13 @@ masterline(Sfio_t* mp, Sfio_t* lp, char* prompt, int must, int timeout, Master_t
 	error(-2, "b \"%s\"", fmtnesq(s, "\"", n));
 	if ((bp->max - bp->end) < n)
 	{
-		a = roundof(bp->max - bp->buf + n, SFIO_BUFSIZE);
+		ssize_t	new_buf_size;
 		r = bp->buf;
-		if (!(bp->buf = vmnewof(bp->vm, bp->buf, char, a, 0)))
+		new_buf_size = roundof(bp->max - bp->buf + n, SFIO_BUFSIZE);
+		bp->buf = realloc(bp->buf, new_buf_size);
+		if (!bp->buf)
 			outofmemory();
-		bp->max = bp->buf + a;
+		bp->max = bp->buf + new_buf_size;
 		if (bp->buf != r)
 		{
 			d = bp->buf - r;
@@ -781,18 +780,15 @@ dialogue(Sfio_t* mp, Sfio_t* lp, int delay, int timeout)
 	char*		m;
 	char*		e;
 	char*		id;
-	Vmalloc_t*	vm;
 	Cond_t*		cond;
 	Master_t*	master;
 
 	int		status = 0;
 
-	if (!(vm = vmopen(Vmdcheap, Vmbest, 0)) ||
-	    !(cond = vmnewof(vm, 0, Cond_t, 1, 0)) ||
-	    !(master = vmnewof(vm, 0, Master_t, 1, 0)) ||
-	    !(master->buf = vmnewof(vm, 0, char, 2 * SFIO_BUFSIZE, 0)))
+	if (!(cond = calloc(1, sizeof(*cond))) ||
+	    !(master = calloc(1, sizeof(*master))) ||
+	    !(master->buf = calloc(2 * SFIO_BUFSIZE, sizeof(char))))
 		outofmemory();
-	master->vm = vm;
 	master->cur = master->end = master->buf;
 	master->max = master->buf + 2 * SFIO_BUFSIZE - 1;
 	master->restore = -1;
@@ -841,7 +837,7 @@ dialogue(Sfio_t* mp, Sfio_t* lp, int delay, int timeout)
 				error(2, "%s: invalid delay -- milliseconds expected", s);
 			break;
 		case 'i':
-			if (!cond->next && !(cond->next = vmnewof(vm, 0, Cond_t, 1, 0)))
+			if (!cond->next && !(cond->next = calloc(1, sizeof(Cond_t))))
 				outofmemory();
 			cond = cond->next;
 			cond->flags = IF;
@@ -946,38 +942,29 @@ dialogue(Sfio_t* mp, Sfio_t* lp, int delay, int timeout)
 		case 'I':
 			if (master->ignore)
 			{
-				vmfree(vm, master->ignore);
+				free(master->ignore);
 				master->ignore = 0;
 			}
-			if (*s && !(master->ignore = vmstrdup(vm, s)))
-			{
-				error(ERROR_SYSTEM|2, "out of memory");
-				goto done;
-			}
+			if (*s && !(master->ignore = strdup(s)))
+				outofmemory();
 			break;
 		case 'L':
 			if (error_info.id)
 			{
-				vmfree(vm, error_info.id);
+				free(error_info.id);
 				error_info.id = 0;
 			}
-			if (*s && !(error_info.id = vmstrdup(vm, s)))
-			{
-				error(ERROR_SYSTEM|2, "out of memory");
-				goto done;
-			}
+			if (*s && !(error_info.id = strdup(s)))
+				outofmemory();
 			break;
 		case 'P':
 			if (master->prompt)
 			{
-				vmfree(vm, master->prompt);
+				free(master->prompt);
 				master->prompt = 0;
 			}
-			if (*s && !(master->prompt = vmstrdup(vm, s)))
-			{
-				error(ERROR_SYSTEM|2, "out of memory");
-				goto done;
-			}
+			if (*s && !(master->prompt = strdup(s)))
+				outofmemory();
 			break;
 		default:
 			if (cond->flags & SKIP)
@@ -993,8 +980,6 @@ dialogue(Sfio_t* mp, Sfio_t* lp, int delay, int timeout)
 		sfclose(mp);
 	error_info.id = id;
 	error_info.line = line;
-	if (vm)
-		vmclose(vm);
 	return status ? status : error_info.errors != 0;
 }
 

src/cmd/ksh93/Mamfile

@@ -262,7 +262,6 @@ make install virtual
 	done
 
 	make include/jobs.h
-		prev %{INCLUDE_AST}/vmalloc.h
 		prev include/terminal.h
 		prev %{INCLUDE_AST}/aso.h
 		prev %{INCLUDE_AST}/sfio.h
@@ -631,7 +630,6 @@ make install virtual
 	done
 
 	make sh/fault.c
-		prev %{INCLUDE_AST}/vmalloc.h
 		prev include/ulimit.h
 		prev include/builtins.h
 		prev include/path.h
@@ -875,7 +873,6 @@ make install virtual
 	done
 
 	make sh/xec.c
-		prev %{INCLUDE_AST}/vmalloc.h
 		prev include/streval.h
 		prev FEATURE/locale
 		prev FEATURE/externs

src/cmd/ksh93/README

@@ -28,8 +28,7 @@ The options have the following defaults and meanings:
                      See README-AUDIT.md.
 
     ALL_LIBCMD   off Enables all of the available /opt/ast/bin builtin commands
-                     in src/lib/libcmd (excludes vmstate if vmalloc is not
-                     enabled).
+                     in src/lib/libcmd.
 
     AUDIT        on  For auditing specific users.
                      Noted by "A" in the version string when enabled.

src/cmd/ksh93/SHOPT.sh

@@ -7,7 +7,7 @@
 
 SHOPT ACCT=0				# accounting
 SHOPT ACCTFILE=0			# per-user accounting info
-SHOPT ALL_LIBCMD=0			# enable all available builtins in src/lib/libcmd (excludes vmstate if vmalloc isn't enabled)
+SHOPT ALL_LIBCMD=0			# enable all available builtins in src/lib/libcmd
 SHOPT AUDIT=1				# enable auditing per SHOPT_AUDITFILE
 SHOPT AUDITFILE='"/etc/ksh_audit"'	# auditing file
 SHOPT BGX=1				# one SIGCHLD trap per completed job

src/cmd/ksh93/data/builtins.c

@@ -174,9 +174,6 @@ const struct shtable3 shtab_builtins[] =
 	CMDLIST(ln)
 	CMDLIST(mktemp)
 	CMDLIST(mv)
-#if !_std_malloc && !_AST_std_malloc
-	CMDLIST(vmstate)  /* vmstate only works with vmalloc */
-#endif
 #endif
 #if SHOPT_REGRESS
 	"__regress__",		NV_BLTIN|BLT_ENV,	bltin(__regress__),

src/cmd/ksh93/include/jobs.h

@@ -95,22 +95,11 @@ struct jobs
 
 extern struct jobs job;
 
-#if !_std_malloc
-#include <vmalloc.h>
-#ifdef vmlocked
-#define vmbusy()	vmlocked(Vmregion)
-#else
-#define vmbusy()	(vmstat(0,0)!=0)
-#endif
-#else
-#define vmbusy()	0
-#endif
-
 #define job_lock()	asoincint(&job.in_critical)
 #define job_unlock()	\
 	do { \
 		int	_sig; \
-		if (asogetint(&job.in_critical) == 1 && (_sig = job.savesig) && !vmbusy()) \
+		if (asogetint(&job.in_critical) == 1 && (_sig = job.savesig)) \
 		    job_reap(_sig); \
 		asodecint(&job.in_critical); \
 	} while(0)

src/cmd/ksh93/sh/fault.c

@@ -40,21 +40,6 @@
 static char	indone;
 static int	cursig = -1;
 
-#if !_std_malloc
-#   include	<vmalloc.h>
-#endif
-#if  defined(VMFL)
-    /*
-     * This exception handler is called after vmalloc() unlocks the region
-     */
-    static int malloc_done(Vmalloc_t* vm, int type, void* val, Vmdisc_t* dp)
-    {
-	dp->exceptf = 0;
-	sh_exit(SH_EXITSIG);
-	return 0;
-    }
-#endif
-
 /*
  * Most signals caught or ignored by the shell come here
 */
@@ -145,16 +130,6 @@ void	sh_fault(int sig)
 			sh.trapnote |= SH_SIGSET;
 			if(sig <= sh.sigmax)
 				sh.sigflag[sig] |= SH_SIGSET;
-#if  defined(VMFL)
-			if(abortsig(sig))
-			{
-				/* abort inside malloc, process when malloc returns */
-				/* VMFL defined when using vmalloc() */
-				Vmdisc_t* dp = vmdisc(Vmregion,0);
-				if(dp)
-					dp->exceptf = malloc_done;
-			}
-#endif
 			goto done;
 		}
 	}

src/cmd/ksh93/sh/init.c

@@ -140,10 +140,6 @@ char e_version[]	= "\n@(#)$Id: Version "
 #define ATTRS		1
 			"s"
 #endif
-#if !_std_malloc && !_AST_std_malloc
-#define ATTRS		1
-			"v"	/* uses vmalloc */
-#endif
 #if ATTRS
 			" "
 #endif
@@ -243,47 +239,53 @@ static noreturn void *nomemory(size_t s)
  */
 void *sh_malloc(size_t size)
 {
-	void *cp = malloc(size);
+	void *cp;
+	cp = malloc(size);
 	if(!cp)
 		nomemory(size);
 	return cp;
 }
 
 void *sh_realloc(void *ptr, size_t size)
 {
-	void *cp = realloc(ptr, size);
+	void *cp;
+	cp = realloc(ptr, size);
 	if(!cp)
 		nomemory(size);
 	return cp;
 }
 
 void *sh_calloc(size_t nmemb, size_t size)
 {
-	void *cp = calloc(nmemb, size);
+	void *cp;
+	cp = calloc(nmemb, size);
 	if(!cp)
 		nomemory(size);
 	return cp;
 }
 
 char *sh_strdup(const char *s)
 {
-	char *dup = strdup(s);
+	char *dup;
+	dup = strdup(s);
 	if(!dup)
 		nomemory(strlen(s)+1);
 	return dup;
 }
 
 void *sh_memdup(const void *s, size_t n)
 {
-	void *dup = memdup(s, n);
+	void *dup;
+	dup = memdup(s, n);
 	if(!dup)
 		nomemory(n);
 	return dup;
 }
 
 char *sh_getcwd(void)
 {
-	char *cwd = getcwd(NULL, 0);
+	char *cwd;
+	cwd = getcwd(NULL, 0);
 	if(!cwd && errno==ENOMEM)
 		nomemory(PATH_MAX);
 	return cwd;
@@ -1277,9 +1279,6 @@ Shell_t *sh_init(int argc,char *argv[], Shinit_f userinit)
 	int type = 0;
 	static char *login_files[2];
 	sh_onstate(SH_INIT);
-#if !_std_malloc
-	memfatal();
-#endif
 	n = strlen(e_version);
 	if(e_version[n-1]=='$' && e_version[n-2]==' ')
 		e_version[n-2]=0;

src/cmd/ksh93/sh/jobs.c

@@ -264,12 +264,6 @@ int job_reap(int sig)
 	int nochild = 0, oerrno = errno, wstat;
 	Waitevent_f waitevent = sh.waitevent;
 	static int wcontinued = WCONTINUED;
-	if (vmbusy())
-	{
-		errormsg(SH_DICT,ERROR_warn(0),"vmbusy() inside job_reap() -- should not happen");
-		if (getenv("_AST_KSH_VMBUSY_ABORT"))
-			abort();
-	}
 #ifdef DEBUG
 	if(sfprintf(sfstderr,"ksh: job line %4d: reap PID=%lld critical=%d signal=%d\n",__LINE__,(Sflong_t)sh.current_pid,job.in_critical,sig) <=0)
 		write(2,"waitsafe\n",9);
@@ -490,7 +484,7 @@ int job_reap(int sig)
  */
 static void job_waitsafe(int sig)
 {
-	if(job.in_critical || vmbusy())
+	if(job.in_critical)
 	{
 		job.savesig = sig;
 		job.waitsafe++;

src/cmd/ksh93/sh/pmain.c

@@ -2,7 +2,7 @@
 *                                                                      *
 *               This software is part of the ast package               *
 *          Copyright (c) 1982-2011 AT&T Intellectual Property          *
-*          Copyright (c) 2020-2023 Contributors to ksh 93u+m           *
+*          Copyright (c) 2020-2024 Contributors to ksh 93u+m           *
 *                      and is licensed under the                       *
 *                 Eclipse Public License, Version 2.0                  *
 *                                                                      *
@@ -21,7 +21,6 @@
 #include	"FEATURE/externs"
 
 #if defined(__sun) && _sys_mman && _lib_memcntl && defined(MHA_MAPSIZE_STACK) && defined(MC_HAT_ADVISE)
-#   undef	VM_FLAGS	/* Solaris vs vmalloc.h symbol clash */
 #   include	<sys/mman.h>
 #else
 #   undef	_lib_memcntl