[quake3] [PATCH] better NX support

[James Lacey]

It will break the Windows build as mmap() is not in the CRT. Unless someone
has an mmap() wrapper for the Win32 API (which is probably out there
somewhere), equivalent code would have to be written using the Win32 memory
mapped file functions.

James

-----Original Message-----
From: Ludwig Nussel [mailto:ludwig.nussel at suse.de] 
Sent: Monday, February 27, 2006 7:00 AM
To: quake3 at icculus.org
Subject: [quake3] [PATCH] better NX support

Hi,

Following patch uses anonymous mmap for the code produced by the vm compiler
on
i386 and x86_64. This way it's possible to have the memory only executable
rather than executable and writeable. It's a cleaner method than using
mprotect
on memory allocated by Hunk_Alloc IMO. Changing the protection bits in
vm_x86
is necessary otherwise quake3 will segfault systems where NX is enabled. I
did
not commit the patch yet, I'd like to get an explicit OK first. I don't know
if
it breaks the Windows build.

cu
Ludwig

Index: code/qcommon/vm_x86.c
===================================================================
--- code/qcommon/vm_x86.c	(Revision 592)
+++ code/qcommon/vm_x86.c	(Arbeitskopie)
@@ -31,6 +31,8 @@
 #include <sys/mman.h> // for PROT_ stuff
 #endif
 
+static void VM_Destroy_Compiled(vm_t* self);
+
 /*
 
   eax	scratch
@@ -1069,34 +1071,29 @@
 
 	// copy to an exact size buffer on the hunk
 	vm->codeLength = compiledOfs;
-	vm->codeBase = Hunk_Alloc( compiledOfs, h_low );
+	vm->codeBase = mmap(NULL, compiledOfs, PROT_WRITE,
MAP_SHARED|MAP_ANONYMOUS, -1, 0);
+	if(vm->codeBase == (void*)-1)
+		Com_Error(ERR_DROP, "VM_CompileX86: can't mmap memory");
+
 	Com_Memcpy( vm->codeBase, buf, compiledOfs );
+	if(mprotect(vm->codeBase, compiledOfs, PROT_EXEC))
+		Com_Error(ERR_DROP, "VM_CompileX86: mprotect failed");
+
 	Z_Free( buf );
 	Z_Free( jused );
 	Com_Printf( "VM file %s compiled to %i bytes of code\n", vm->name,
compiledOfs );
 
+	vm->destroy = VM_Destroy_Compiled;
+
 	// offset all the instruction pointers for the new location
 	for ( i = 0 ; i < header->instructionCount ; i++ ) {
 		vm->instructionPointers[i] += (int)vm->codeBase;
 	}
+}
 
-#if 0 // ndef _WIN32
-	// Must make the newly generated code executable
-	{
-		int r;
-		unsigned long addr;
-		int psize = getpagesize();
-
-		addr = ((int)vm->codeBase & ~(psize-1)) - psize;
-
-		r = mprotect((char*)addr, vm->codeLength + (int)vm->codeBase
- addr + psize, 
-			PROT_READ | PROT_WRITE | PROT_EXEC );
-
-		if (r < 0)
-			Com_Error( ERR_FATAL, "mprotect failed to change
PROT_EXEC" );
-	}
-#endif
-
+void VM_Destroy_Compiled(vm_t* self)
+{
+	munmap(self->codeBase, self->codeLength);
 }
 
 /*
Index: code/qcommon/vm_x86_64.c
===================================================================
--- code/qcommon/vm_x86_64.c	(Revision 592)
+++ code/qcommon/vm_x86_64.c	(Arbeitskopie)
@@ -39,6 +39,8 @@
 #define Dfprintf(args...)
 #endif
 
+static void VM_Destroy_Compiled(vm_t* self);
+
 /*
 
   eax	scratch
@@ -309,7 +311,7 @@
 static int doas(char* in, char* out, unsigned char** compiledcode)
 {
 	char* buf;
-	char* mem;
+	unsigned char* mem;
 	size_t size = -1, allocsize;
 	int ps;
 	pid_t pid;
@@ -365,35 +367,9 @@
 		return -1;
 	}
 
-	ps = sysconf(_SC_PAGE_SIZE);
-	if(ps == -1)
-	{
-		Com_Printf(S_COLOR_RED "can't determine page size: %s\n",
strerror(errno));
-		return -1;
-	}
+	*compiledcode = mem;
 
-	--ps;
-
-	allocsize = (size+ps)&~ps;
-	buf = Hunk_Alloc(allocsize, h_high);
-
-	buf = (void*)(((unsigned long)buf+ps)&~ps);
-
-	memcpy(buf, mem, size);
-
-	munmap(mem, 0);
-
-	if((*compiledcode = (unsigned char*)buf))
-	{
-		// need to be able to exec code
-		if(mprotect(buf, allocsize, PROT_READ|PROT_WRITE|PROT_EXEC)
== -1)
-		{
-			Com_Error(ERR_FATAL, "mprotect failed on %p+%x:
%s\n", buf, allocsize, strerror(errno));
-		}
-		return size;
-	}
-
-	return -1;
+	return size;
 }
 
 static void block_copy_vm(unsigned dest, unsigned src, unsigned count)
@@ -897,6 +873,8 @@
 
 	vm->codeBase   = compiledcode; // remember to skip ELF header!
 	vm->codeLength = compiledsize;
+
+	vm->destroy = VM_Destroy_Compiled;
 	
 	entryPoint = getentrypoint(vm);
 
@@ -930,6 +908,12 @@
 	}
 }
 
+
+void VM_Destroy_Compiled(vm_t* self)
+{
+	munmap(self->codeBase, self->codeLength);
+}
+
 /*
 ==============
 VM_CallCompiled
Index: code/qcommon/vm.c
===================================================================
--- code/qcommon/vm.c	(Revision 592)
+++ code/qcommon/vm.c	(Arbeitskopie)
@@ -615,6 +615,9 @@
 */
 void VM_Free( vm_t *vm ) {
 
+	if(vm->destroy)
+		vm->destroy(vm);
+
 	if ( vm->dllHandle ) {
 		Sys_UnloadDll( vm->dllHandle );
 		Com_Memset( vm, 0, sizeof( *vm ) );

-- 
 (o_   Ludwig Nussel
 //\   SUSE LINUX Products GmbH, Development
 V_/_  http://www.suse.de/