diff --git a/docs/support.md b/docs/support.md index e20fce4..01cbb1d 100644 --- a/docs/support.md +++ b/docs/support.md @@ -9,6 +9,10 @@ java-web targets headless Java: console apps, libraries and javac. - `java.desktop`'s headless parts work: 2D geometry, image pixels, `Graphics2D` drawing on a `BufferedImage`, and JavaBeans. So does `java.xml`, including XML Schema validation and DTDs from the JDK's built-in catalog. +- File mappings are copies in the VM's 512 MB native memory, which is also + shared with direct buffers and other native allocations. A mapping that does + not fit fails with `IOException: Map failed` and an `OutOfMemoryError` cause. + Read-write changes reach the file on force or unmap. - `Deflater` writes valid DEFLATE streams using stored blocks. Compression levels and strategies currently do not reduce output size; compressed bytes and compression ratios differ from HotSpot's zlib output. diff --git a/src/memory.ts b/src/memory.ts index e175ac8..c23ed37 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -6,6 +6,7 @@ import { PendingThrow } from "./thread.ts"; const ALIGN = 16; // Like a failed malloc in HotSpot, going past this throws OutOfMemoryError. const LIMIT = 512 << 20; +const MIN_SHRINK = 2 << 20; export class NativeMemory { buf: ArrayBuffer; @@ -15,23 +16,64 @@ export class NativeMemory { private sizes = new Map(); // free blocks by size private free = new Map(); + // end address to size, and start address to list index, for the free blocks + private freeEnds = new Map(); + private freeAt = new Map(); + + private initial: number; constructor(initial = 1 << 20) { + this.initial = initial; this.buf = new ArrayBuffer(initial); this.u8 = new Uint8Array(this.buf); this.dv = new DataView(this.buf); } + private resize(size: number) { + const next = new ArrayBuffer(size); + new Uint8Array(next).set(this.u8.subarray(0, Math.min(this.top, size))); + this.buf = next; + this.u8 = new Uint8Array(next); + this.dv = new DataView(next); + } + private grow(need: number) { if (need > LIMIT) throw new PendingThrow("java/lang/OutOfMemoryError", `Unable to allocate ${need - this.buf.byteLength} bytes`); let size = this.buf.byteLength; while (size < need) size *= 2; size = Math.min(size, LIMIT); - const next = new ArrayBuffer(size); - new Uint8Array(next).set(this.u8); - this.buf = next; - this.u8 = new Uint8Array(next); - this.dv = new DataView(next); + try { this.resize(size); } + catch (e) { + if (e instanceof RangeError) throw new PendingThrow("java/lang/OutOfMemoryError", `Unable to allocate ${size} bytes`); + throw e; + } + } + + // The saved state of a snapshot: the bytes up to top and the block tables. + restore(bytes: Uint8Array, top: number, sizes: Map, free: Map) { + this.buf = new ArrayBuffer(Math.max(this.buf.byteLength, bytes.length)); + this.u8 = new Uint8Array(this.buf); + this.dv = new DataView(this.buf); + this.u8.set(bytes); + this.top = top; + this.sizes = sizes; + this.free = free; + this.freeEnds.clear(); + this.freeAt.clear(); + for (const [size, blocks] of free) { + blocks.forEach((start, i) => { this.freeEnds.set(start + size, size); this.freeAt.set(start, i); }); + } + } + + // The order of a free list does not matter, so the last block fills the gap. + private takeFree(size: number, list: number[], at: number): number { + const addr = list[at]; + const last = list.pop()!; + if (at < list.length) { list[at] = last; this.freeAt.set(last, at); } + if (!list.length) this.free.delete(size); + this.freeAt.delete(addr); + this.freeEnds.delete(addr + size); + return addr; } malloc(n: number, alignment = ALIGN): number { @@ -39,16 +81,14 @@ export class NativeMemory { const list = this.free.get(size); const reuse = list?.findIndex((addr) => addr % alignment === 0) ?? -1; if (list && reuse >= 0) { - // the order of a free list does not matter, so the last block fills the gap - const addr = list[reuse]; - list[reuse] = list[list.length - 1]; - list.pop(); + const addr = this.takeFree(size, list, reuse); this.sizes.set(addr, size); return addr; } const addr = Math.ceil(this.top / alignment) * alignment; - this.top = addr + size; - if (this.top > this.buf.byteLength) this.grow(this.top); + const top = addr + size; + if (top > this.buf.byteLength) this.grow(top); + this.top = top; this.sizes.set(addr, size); return addr; } @@ -57,9 +97,28 @@ export class NativeMemory { const size = this.sizes.get(addr); if (size === undefined) return; this.sizes.delete(addr); - let list = this.free.get(size); - if (!list) this.free.set(size, (list = [])); - list.push(addr); + if (addr + size < this.top) { + let list = this.free.get(size); + if (!list) this.free.set(size, (list = [])); + this.freeAt.set(addr, list.push(addr) - 1); + this.freeEnds.set(addr + size, size); + return; + } + // Lower top over this block and any free blocks that end where it starts. + let top = addr; + for (let below = this.freeEnds.get(top); below !== undefined; below = this.freeEnds.get(top)) { + top -= below; + this.takeFree(below, this.free.get(below)!, this.freeAt.get(top)!); + } + this.top = Math.max(top, ALIGN); + if (this.buf.byteLength >= MIN_SHRINK && this.top <= this.buf.byteLength / 4) { + const size = Math.max(this.initial, 2 ** Math.ceil(Math.log2(this.top))); + // best effort: a free must not fail + if (size < this.buf.byteLength) { + try { this.resize(size); } + catch (e) { if (!(e instanceof RangeError)) throw e; } + } + } } realloc(addr: number, n: number): number { diff --git a/src/natives/nio.ts b/src/natives/nio.ts index 9b0c7e1..9bb212e 100644 --- a/src/natives/nio.ts +++ b/src/natives/nio.ts @@ -333,7 +333,7 @@ function writeBack(t: JThread, m: Mapping, base: number, addr: number, len: numb if (n <= 0) return true; const save = m.f.pos; m.f.pos = at; - m.f.write(t.vm.mem.u8.slice(addr, addr + n), true); + m.f.write(t.vm.mem.u8.subarray(addr, addr + n), true); m.f.pos = save; return flush(t, m.f); } @@ -429,7 +429,13 @@ Object.assign(natives, { // writes only then. [`${UFD}map0(Ljava/io/FileDescriptor;IJJZ)J`]: (t: JThread, [fd, prot, pos, len]: any[]) => { const f = chanFile(t, fd); - const addr = t.vm.mem.malloc(Number(len), 4096); + let addr: number; + try { addr = t.vm.mem.malloc(Number(len), 4096); } + catch (e) { + // FileChannelImpl retries once, then wraps this in IOException("Map failed") + if (e instanceof PendingThrow) throw new PendingThrow("java/lang/OutOfMemoryError", "Map failed"); + throw e; + } const save = f.pos; f.pos = Number(pos); const n = Math.min(f.size - Number(pos), Number(len)); diff --git a/src/snapshot.ts b/src/snapshot.ts index d2661dd..7f9638b 100644 --- a/src/snapshot.ts +++ b/src/snapshot.ts @@ -417,14 +417,7 @@ export function restoreVM(vm: JVM, image: SnapshotImage, main: JThread) { vm.state.set(k, { next: fds.next, files }); } else vm.state.set(k, d.dec(x)); } - const mem = vm.mem as any; - mem.buf = new ArrayBuffer(Math.max(mem.buf.byteLength, v.mem.buf.length)); - mem.u8 = new Uint8Array(mem.buf); - mem.dv = new DataView(mem.buf); - mem.u8.set(v.mem.buf); - mem.top = v.mem.top; - mem.sizes = d.dec(v.mem.sizes); - mem.free = d.dec(v.mem.free); + vm.mem.restore(v.mem.buf, v.mem.top, d.dec(v.mem.sizes), d.dec(v.mem.free)); d.flush(); const mainRec = image.threads[0]; diff --git a/test/java/MappedFile.java b/test/java/MappedFile.java new file mode 100644 index 0000000..506db56 --- /dev/null +++ b/test/java/MappedFile.java @@ -0,0 +1,34 @@ +import java.io.RandomAccessFile; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import sun.misc.Unsafe; +import java.lang.reflect.Field; + +public class MappedFile { + public static void main(String[] args) throws Exception { + Path path = Files.createTempFile("mapped", ".bin"); + try { + try (RandomAccessFile file = new RandomAccessFile(path.toFile(), "rw")) { + file.setLength(4 * 1024 * 1024); + MappedByteBuffer map = file.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, file.length()); + map.put(0, (byte) 17); + map.put(2 * 1024 * 1024, (byte) 42); + map.force(); + map.put(4 * 1024 * 1024 - 1, (byte) 99); + Field field = Unsafe.class.getDeclaredField("theUnsafe"); + field.setAccessible(true); + ((Unsafe) field.get(null)).invokeCleaner(map); + file.seek(0); + System.out.println(file.read()); + file.seek(2 * 1024 * 1024); + System.out.println(file.read()); + file.seek(file.length() - 1); + System.out.println(file.read()); + } + } finally { + Files.delete(path); + } + } +} diff --git a/test/jtreg/passing.txt b/test/jtreg/passing.txt index 17e8cde..2debf74 100644 --- a/test/jtreg/passing.txt +++ b/test/jtreg/passing.txt @@ -1967,6 +1967,7 @@ java/nio/channels/DatagramChannel/ThereCanBeOnlyOne.java java/nio/channels/DatagramChannel/Truncate.java java/nio/channels/DatagramChannel/UseDGWithIPv6.java java/nio/channels/FileChannel/Args.java +java/nio/channels/FileChannel/AtomicAppend.java java/nio/channels/FileChannel/CloseDuringTransfer.java java/nio/channels/FileChannel/ClosedByInterrupt.java java/nio/channels/FileChannel/ClosedChannelTransfer.java diff --git a/test/jtreg/times.tsv b/test/jtreg/times.tsv index 0ae8ee6..169cc3d 100644 --- a/test/jtreg/times.tsv +++ b/test/jtreg/times.tsv @@ -3065,6 +3065,7 @@ 2080 java/lang/ClassLoader/LoadNullClass.java 2078 java/io/Serializable/InvalidObjectException/TestIoeConstructors.java 2077 java/io/Serializable/enum/array/Test.java +2076 java/nio/channels/FileChannel/AtomicAppend.java 2075 java/lang/Byte/Decode.java 2072 java/nio/channels/FileChannel/TransferTo6GBFile.java 2071 java/util/concurrent/forkjoin/AsyncShutdownNow.java diff --git a/test/memory.test.ts b/test/memory.test.ts index ea35af7..29c316f 100644 --- a/test/memory.test.ts +++ b/test/memory.test.ts @@ -10,3 +10,79 @@ test("mapped allocations are page aligned and reusable", () => { mem.freeBlock(page); assert.equal(mem.malloc(128, 4096), page); }); + +test("releases a large top allocation and preserves live blocks", () => { + const mem = new NativeMemory(); + const first = mem.malloc(32); + mem.u8[first] = 123; + const middle = mem.malloc(32); + const large = mem.malloc(16 << 20, 4096); + mem.freeBlock(middle); + mem.freeBlock(large); + assert.equal(mem.buf.byteLength, 1 << 20); + assert.equal(mem.u8[first], 123); + assert.equal(mem.malloc(32), middle); +}); + +test("a failed allocation leaves memory usable", () => { + const mem = new NativeMemory(); + const addr = mem.malloc(32); + assert.throws(() => mem.malloc(512 << 20)); + assert.equal(mem.malloc(32), addr + 32); +}); + +test("freeing blocks in any order lowers top over adjacent free blocks", () => { + const mem = new NativeMemory(); + const a = mem.malloc(64); + const b = mem.malloc(64); + const c = mem.malloc(64); + mem.freeBlock(b); + mem.freeBlock(a); + mem.freeBlock(c); + assert.equal(mem.malloc(64), a); +}); + +test("small frees keep the buffer", () => { + const mem = new NativeMemory(4 << 20); + const buf = mem.buf; + mem.freeBlock(mem.malloc(32)); + assert.equal(mem.buf, buf); +}); + +test("a restored memory lowers top over free blocks", () => { + const mem = new NativeMemory(); + const a = mem.malloc(64); + const b = mem.malloc(64); + const c = mem.malloc(64); + mem.freeBlock(b); + const state = mem as any; + const copy = new NativeMemory(); + copy.restore(state.u8.slice(0, state.top), state.top, new Map(state.sizes), new Map([...state.free].map(([k, v]) => [k, [...v]]))); + copy.freeBlock(c); + copy.freeBlock(a); + assert.equal(copy.malloc(64), a); + assert.equal(copy.malloc(64), b); +}); + +test("freeing many same-sized blocks then the top merges them all", () => { + const mem = new NativeMemory(); + const blocks = Array.from({ length: 20000 }, () => mem.malloc(32)); + const top = mem.malloc(32); + for (let i = blocks.length - 1; i >= 0; i--) mem.freeBlock(blocks[i]); + mem.freeBlock(top); + assert.equal(mem.malloc(32), blocks[0]); + assert.equal(mem.malloc(32), blocks[1]); +}); + +test("reused free blocks keep the merge index consistent", () => { + const mem = new NativeMemory(); + const [a, b, c, d] = [mem.malloc(32), mem.malloc(32), mem.malloc(32), mem.malloc(32)]; + mem.freeBlock(a); + mem.freeBlock(b); + mem.freeBlock(c); + assert.equal(mem.malloc(32), a); + mem.freeBlock(d); + assert.equal(mem.malloc(32), b); + mem.freeBlock(a); + assert.equal(mem.malloc(32), a); +}); diff --git a/test/nio.test.ts b/test/nio.test.ts index 150d0e8..d41e866 100644 --- a/test/nio.test.ts +++ b/test/nio.test.ts @@ -9,6 +9,8 @@ import { natives } from "../src/natives/nio.ts"; import { PendingThrow } from "../src/thread.ts"; import type { JThread } from "../src/thread.ts"; import { NodeBackend } from "../src/node.ts"; +import { NativeMemory } from "../src/memory.ts"; +import { OpenFile } from "../src/natives/io.ts"; function thread(backend: Backend): JThread { const fs = new VFS(); @@ -122,3 +124,18 @@ test("a large file on a read-only host mount uses a handle", async () => { rmSync(dir, { recursive: true, force: true }); } }); + +test("map0 of a region that does not fit throws OutOfMemoryError for FileChannelImpl to wrap", () => { + const backend = memory(); + const t = thread(backend); + const vm = t.vm as any; + vm.mem = new NativeMemory(); + vm.state.set("fds", { files: new Map([[3, new OpenFile("/file", new Uint8Array(1), false)]]) }); + vm.getField = () => 3; + assert.throws(() => natives["sun/nio/ch/UnixFileDispatcherImpl.map0(Ljava/io/FileDescriptor;IJJZ)J"](t, [{}, 0, 0n, 600n << 20n]), (e) => { + assert.ok(e instanceof PendingThrow); + assert.equal(e.cls, "java/lang/OutOfMemoryError"); + assert.equal(e.msg, "Map failed"); + return true; + }); +});