Give native memory back after unmapping, and fail oversized mappings like HotSpot

docs/support.md

@@ -9,6 +9,10 @@ java-web targets headless Java: console apps, libraries and javac.
 - `java.desktop`'s headless parts work: 2D geometry, image pixels, `Graphics2D`
   drawing on a `BufferedImage`, and JavaBeans. So does `java.xml`, including
   XML Schema validation and DTDs from the JDK's built-in catalog.
+- File mappings are copies in the VM's 512 MB native memory, which is also
+  shared with direct buffers and other native allocations. A mapping that does
+  not fit fails with `IOException: Map failed` and an `OutOfMemoryError` cause.
+  Read-write changes reach the file on force or unmap.
 - `Deflater` writes valid DEFLATE streams using stored blocks. Compression
   levels and strategies currently do not reduce output size; compressed bytes
   and compression ratios differ from HotSpot's zlib output.

src/memory.ts

@@ -6,6 +6,7 @@ import { PendingThrow } from "./thread.ts";
 const ALIGN = 16;
 // Like a failed malloc in HotSpot, going past this throws OutOfMemoryError.
 const LIMIT = 512 << 20;
+const MIN_SHRINK = 2 << 20;
 
 export class NativeMemory {
   buf: ArrayBuffer;
@@ -15,40 +16,79 @@ export class NativeMemory {
   private sizes = new Map<number, number>();
   // free blocks by size
   private free = new Map<number, number[]>();
+  // end address to size, and start address to list index, for the free blocks
+  private freeEnds = new Map<number, number>();
+  private freeAt = new Map<number, number>();
+
+  private initial: number;
 
   constructor(initial = 1 << 20) {
+    this.initial = initial;
     this.buf = new ArrayBuffer(initial);
     this.u8 = new Uint8Array(this.buf);
     this.dv = new DataView(this.buf);
   }
 
+  private resize(size: number) {
+    const next = new ArrayBuffer(size);
+    new Uint8Array(next).set(this.u8.subarray(0, Math.min(this.top, size)));
+    this.buf = next;
+    this.u8 = new Uint8Array(next);
+    this.dv = new DataView(next);
+  }
+
   private grow(need: number) {
     if (need > LIMIT) throw new PendingThrow("java/lang/OutOfMemoryError", `Unable to allocate ${need - this.buf.byteLength} bytes`);
     let size = this.buf.byteLength;
     while (size < need) size *= 2;
     size = Math.min(size, LIMIT);
-    const next = new ArrayBuffer(size);
-    new Uint8Array(next).set(this.u8);
-    this.buf = next;
-    this.u8 = new Uint8Array(next);
-    this.dv = new DataView(next);
+    try { this.resize(size); }
+    catch (e) {
+      if (e instanceof RangeError) throw new PendingThrow("java/lang/OutOfMemoryError", `Unable to allocate ${size} bytes`);
+      throw e;
+    }
+  }
+
+  // The saved state of a snapshot: the bytes up to top and the block tables.
+  restore(bytes: Uint8Array, top: number, sizes: Map<number, number>, free: Map<number, number[]>) {
+    this.buf = new ArrayBuffer(Math.max(this.buf.byteLength, bytes.length));
+    this.u8 = new Uint8Array(this.buf);
+    this.dv = new DataView(this.buf);
+    this.u8.set(bytes);
+    this.top = top;
+    this.sizes = sizes;
+    this.free = free;
+    this.freeEnds.clear();
+    this.freeAt.clear();
+    for (const [size, blocks] of free) {
+      blocks.forEach((start, i) => { this.freeEnds.set(start + size, size); this.freeAt.set(start, i); });
+    }
+  }
+
+  // The order of a free list does not matter, so the last block fills the gap.
+  private takeFree(size: number, list: number[], at: number): number {
+    const addr = list[at];
+    const last = list.pop()!;
+    if (at < list.length) { list[at] = last; this.freeAt.set(last, at); }
+    if (!list.length) this.free.delete(size);
+    this.freeAt.delete(addr);
+    this.freeEnds.delete(addr + size);
+    return addr;
   }
 
   malloc(n: number, alignment = ALIGN): number {
     const size = Math.max(ALIGN, Math.ceil(n / ALIGN) * ALIGN);
     const list = this.free.get(size);
     const reuse = list?.findIndex((addr) => addr % alignment === 0) ?? -1;
     if (list && reuse >= 0) {
-      // the order of a free list does not matter, so the last block fills the gap
-      const addr = list[reuse];
-      list[reuse] = list[list.length - 1];
-      list.pop();
+      const addr = this.takeFree(size, list, reuse);
       this.sizes.set(addr, size);
       return addr;
     }
     const addr = Math.ceil(this.top / alignment) * alignment;
-    this.top = addr + size;
-    if (this.top > this.buf.byteLength) this.grow(this.top);
+    const top = addr + size;
+    if (top > this.buf.byteLength) this.grow(top);
+    this.top = top;
     this.sizes.set(addr, size);
     return addr;
   }
@@ -57,9 +97,28 @@ export class NativeMemory {
     const size = this.sizes.get(addr);
     if (size === undefined) return;
     this.sizes.delete(addr);
-    let list = this.free.get(size);
-    if (!list) this.free.set(size, (list = []));
-    list.push(addr);
+    if (addr + size < this.top) {
+      let list = this.free.get(size);
+      if (!list) this.free.set(size, (list = []));
+      this.freeAt.set(addr, list.push(addr) - 1);
+      this.freeEnds.set(addr + size, size);
+      return;
+    }
+    // Lower top over this block and any free blocks that end where it starts.
+    let top = addr;
+    for (let below = this.freeEnds.get(top); below !== undefined; below = this.freeEnds.get(top)) {
+      top -= below;
+      this.takeFree(below, this.free.get(below)!, this.freeAt.get(top)!);
+    }
+    this.top = Math.max(top, ALIGN);
+    if (this.buf.byteLength >= MIN_SHRINK && this.top <= this.buf.byteLength / 4) {
+      const size = Math.max(this.initial, 2 ** Math.ceil(Math.log2(this.top)));
+      // best effort: a free must not fail
+      if (size < this.buf.byteLength) {
+        try { this.resize(size); }
+        catch (e) { if (!(e instanceof RangeError)) throw e; }
+      }
+    }
   }
 
   realloc(addr: number, n: number): number {

src/natives/nio.ts

@@ -333,7 +333,7 @@ function writeBack(t: JThread, m: Mapping, base: number, addr: number, len: numb
   if (n <= 0) return true;
   const save = m.f.pos;
   m.f.pos = at;
-  m.f.write(t.vm.mem.u8.slice(addr, addr + n), true);
+  m.f.write(t.vm.mem.u8.subarray(addr, addr + n), true);
   m.f.pos = save;
   return flush(t, m.f);
 }
@@ -429,7 +429,13 @@ Object.assign(natives, {
   // writes only then.
   [`${UFD}map0(Ljava/io/FileDescriptor;IJJZ)J`]: (t: JThread, [fd, prot, pos, len]: any[]) => {
     const f = chanFile(t, fd);
-    const addr = t.vm.mem.malloc(Number(len), 4096);
+    let addr: number;
+    try { addr = t.vm.mem.malloc(Number(len), 4096); }
+    catch (e) {
+      // FileChannelImpl retries once, then wraps this in IOException("Map failed")
+      if (e instanceof PendingThrow) throw new PendingThrow("java/lang/OutOfMemoryError", "Map failed");
+      throw e;
+    }
     const save = f.pos;
     f.pos = Number(pos);
     const n = Math.min(f.size - Number(pos), Number(len));

src/snapshot.ts

@@ -417,14 +417,7 @@ export function restoreVM(vm: JVM, image: SnapshotImage, main: JThread) {
       vm.state.set(k, { next: fds.next, files });
     } else vm.state.set(k, d.dec(x));
   }
-  const mem = vm.mem as any;
-  mem.buf = new ArrayBuffer(Math.max(mem.buf.byteLength, v.mem.buf.length));
-  mem.u8 = new Uint8Array(mem.buf);
-  mem.dv = new DataView(mem.buf);
-  mem.u8.set(v.mem.buf);
-  mem.top = v.mem.top;
-  mem.sizes = d.dec(v.mem.sizes);
-  mem.free = d.dec(v.mem.free);
+  vm.mem.restore(v.mem.buf, v.mem.top, d.dec(v.mem.sizes), d.dec(v.mem.free));
   d.flush();
 
   const mainRec = image.threads[0];

test/java/MappedFile.java

@@ -0,0 +1,34 @@
+import java.io.RandomAccessFile;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import sun.misc.Unsafe;
+import java.lang.reflect.Field;
+
+public class MappedFile {
+    public static void main(String[] args) throws Exception {
+        Path path = Files.createTempFile("mapped", ".bin");
+        try {
+            try (RandomAccessFile file = new RandomAccessFile(path.toFile(), "rw")) {
+                file.setLength(4 * 1024 * 1024);
+                MappedByteBuffer map = file.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, file.length());
+                map.put(0, (byte) 17);
+                map.put(2 * 1024 * 1024, (byte) 42);
+                map.force();
+                map.put(4 * 1024 * 1024 - 1, (byte) 99);
+                Field field = Unsafe.class.getDeclaredField("theUnsafe");
+                field.setAccessible(true);
+                ((Unsafe) field.get(null)).invokeCleaner(map);
+                file.seek(0);
+                System.out.println(file.read());
+                file.seek(2 * 1024 * 1024);
+                System.out.println(file.read());
+                file.seek(file.length() - 1);
+                System.out.println(file.read());
+            }
+        } finally {
+            Files.delete(path);
+        }
+    }
+}

test/jtreg/passing.txt

@@ -1967,6 +1967,7 @@ java/nio/channels/DatagramChannel/ThereCanBeOnlyOne.java
 java/nio/channels/DatagramChannel/Truncate.java
 java/nio/channels/DatagramChannel/UseDGWithIPv6.java
 java/nio/channels/FileChannel/Args.java
+java/nio/channels/FileChannel/AtomicAppend.java
 java/nio/channels/FileChannel/CloseDuringTransfer.java
 java/nio/channels/FileChannel/ClosedByInterrupt.java
 java/nio/channels/FileChannel/ClosedChannelTransfer.java

test/jtreg/times.tsv

@@ -3065,6 +3065,7 @@
 2080	java/lang/ClassLoader/LoadNullClass.java
 2078	java/io/Serializable/InvalidObjectException/TestIoeConstructors.java
 2077	java/io/Serializable/enum/array/Test.java
+2076	java/nio/channels/FileChannel/AtomicAppend.java
 2075	java/lang/Byte/Decode.java
 2072	java/nio/channels/FileChannel/TransferTo6GBFile.java
 2071	java/util/concurrent/forkjoin/AsyncShutdownNow.java

test/memory.test.ts

@@ -10,3 +10,79 @@ test("mapped allocations are page aligned and reusable", () => {
   mem.freeBlock(page);
   assert.equal(mem.malloc(128, 4096), page);
 });
+
+test("releases a large top allocation and preserves live blocks", () => {
+  const mem = new NativeMemory();
+  const first = mem.malloc(32);
+  mem.u8[first] = 123;
+  const middle = mem.malloc(32);
+  const large = mem.malloc(16 << 20, 4096);
+  mem.freeBlock(middle);
+  mem.freeBlock(large);
+  assert.equal(mem.buf.byteLength, 1 << 20);
+  assert.equal(mem.u8[first], 123);
+  assert.equal(mem.malloc(32), middle);
+});
+
+test("a failed allocation leaves memory usable", () => {
+  const mem = new NativeMemory();
+  const addr = mem.malloc(32);
+  assert.throws(() => mem.malloc(512 << 20));
+  assert.equal(mem.malloc(32), addr + 32);
+});
+
+test("freeing blocks in any order lowers top over adjacent free blocks", () => {
+  const mem = new NativeMemory();
+  const a = mem.malloc(64);
+  const b = mem.malloc(64);
+  const c = mem.malloc(64);
+  mem.freeBlock(b);
+  mem.freeBlock(a);
+  mem.freeBlock(c);
+  assert.equal(mem.malloc(64), a);
+});
+
+test("small frees keep the buffer", () => {
+  const mem = new NativeMemory(4 << 20);
+  const buf = mem.buf;
+  mem.freeBlock(mem.malloc(32));
+  assert.equal(mem.buf, buf);
+});
+
+test("a restored memory lowers top over free blocks", () => {
+  const mem = new NativeMemory();
+  const a = mem.malloc(64);
+  const b = mem.malloc(64);
+  const c = mem.malloc(64);
+  mem.freeBlock(b);
+  const state = mem as any;
+  const copy = new NativeMemory();
+  copy.restore(state.u8.slice(0, state.top), state.top, new Map(state.sizes), new Map([...state.free].map(([k, v]) => [k, [...v]])));
+  copy.freeBlock(c);
+  copy.freeBlock(a);
+  assert.equal(copy.malloc(64), a);
+  assert.equal(copy.malloc(64), b);
+});
+
+test("freeing many same-sized blocks then the top merges them all", () => {
+  const mem = new NativeMemory();
+  const blocks = Array.from({ length: 20000 }, () => mem.malloc(32));
+  const top = mem.malloc(32);
+  for (let i = blocks.length - 1; i >= 0; i--) mem.freeBlock(blocks[i]);
+  mem.freeBlock(top);
+  assert.equal(mem.malloc(32), blocks[0]);
+  assert.equal(mem.malloc(32), blocks[1]);
+});
+
+test("reused free blocks keep the merge index consistent", () => {
+  const mem = new NativeMemory();
+  const [a, b, c, d] = [mem.malloc(32), mem.malloc(32), mem.malloc(32), mem.malloc(32)];
+  mem.freeBlock(a);
+  mem.freeBlock(b);
+  mem.freeBlock(c);
+  assert.equal(mem.malloc(32), a);
+  mem.freeBlock(d);
+  assert.equal(mem.malloc(32), b);
+  mem.freeBlock(a);
+  assert.equal(mem.malloc(32), a);
+});

test/nio.test.ts

@@ -9,6 +9,8 @@ import { natives } from "../src/natives/nio.ts";
 import { PendingThrow } from "../src/thread.ts";
 import type { JThread } from "../src/thread.ts";
 import { NodeBackend } from "../src/node.ts";
+import { NativeMemory } from "../src/memory.ts";
+import { OpenFile } from "../src/natives/io.ts";
 
 function thread(backend: Backend): JThread {
   const fs = new VFS();
@@ -122,3 +124,18 @@ test("a large file on a read-only host mount uses a handle", async () => {
     rmSync(dir, { recursive: true, force: true });
   }
 });
+
+test("map0 of a region that does not fit throws OutOfMemoryError for FileChannelImpl to wrap", () => {
+  const backend = memory();
+  const t = thread(backend);
+  const vm = t.vm as any;
+  vm.mem = new NativeMemory();
+  vm.state.set("fds", { files: new Map([[3, new OpenFile("/file", new Uint8Array(1), false)]]) });
+  vm.getField = () => 3;
+  assert.throws(() => natives["sun/nio/ch/UnixFileDispatcherImpl.map0(Ljava/io/FileDescriptor;IJJZ)J"](t, [{}, 0, 0n, 600n << 20n]), (e) => {
+    assert.ok(e instanceof PendingThrow);
+    assert.equal(e.cls, "java/lang/OutOfMemoryError");
+    assert.equal(e.msg, "Map failed");
+    return true;
+  });
+});