From 7cb565fe322ada29ab7152db3db358c595f246ff Mon Sep 17 00:00:00 2001 From: Project516 <138796702+Project516@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:23:14 -0600 Subject: [PATCH 1/2] Stream sparse file writes without allocating their gaps. --- src/fs.ts | 6 ++-- src/natives/io.ts | 34 +++++++++++++++++------ src/natives/nio.ts | 4 +-- test/java/StreamingFile.java | 32 ++++++++++++++++++++++ test/jtreg/passing.txt | 11 ++++++++ test/jtreg/times.tsv | 11 ++++++++ test/nio.test.ts | 53 +++++++++++++++++++++++++++++++++++- 7 files changed, 137 insertions(+), 14 deletions(-) diff --git a/src/fs.ts b/src/fs.ts index 10aeb10..9253812 100644 --- a/src/fs.ts +++ b/src/fs.ts @@ -297,14 +297,14 @@ export class VFS { return data; } - canOpenFile(path: string): boolean { + canOpenFile(path: string, write: boolean): boolean { const r = this.resolve(path); - return !!r && r[0].writable && !!r[0].openFile; + return !!r && (!write || r[0].writable) && !!r[0].openFile; } openFile(path: string, write: boolean, truncate: boolean, readable: boolean): FileHandle | undefined { const r = this.resolve(path); - if (!r || !r[0].writable || !r[0].openFile) return undefined; + if (!r || ((write || truncate) && !r[0].writable) || !r[0].openFile) return undefined; const h = r[0].openFile(r[1], write, truncate, readable); // writes through the handle are changes watchers see, as writes through VFS.write are return { diff --git a/src/natives/io.ts b/src/natives/io.ts index 094dc38..8424323 100644 --- a/src/natives/io.ts +++ b/src/natives/io.ts @@ -54,7 +54,7 @@ const closesHosts = new WeakSet(); // A snapshot cannot hold a host handle, nor the opener a file may still need. export function streams(f: OpenFile): boolean { - return !!f.contents.host || hostOpeners.has(f.contents); + return f instanceof DirectoryFile || !!f.contents.host || hostOpeners.has(f.contents); } // Host handles of files the program leaves open close when its VM exits. @@ -120,7 +120,8 @@ export class OpenFile { return n; } - write(src: Uint8Array) { + write(src: Uint8Array, atPosition = false) { + if (this.append && !atPosition) this.pos = this.size; const host = this.contents.host; if (host) { hostCall(() => host.write(src, this.pos)); @@ -128,9 +129,10 @@ export class OpenFile { if (this.pos > this.size) this.size = this.pos; return; } + const end = this.pos + src.length; + if (end >= STREAM_MIN && this.stream()) return this.write(src, atPosition); // past the end, the gap reads as zeros (the buffer may hold bytes from before a truncation) if (this.pos > this.size) this.data.fill(0, this.size, Math.min(this.pos, this.data.length)); - const end = this.pos + src.length; if (end > this.data.length) { const next = new Uint8Array(Math.max(end, this.data.length * 2, 64)); next.set(this.data.subarray(0, this.size)); @@ -140,7 +142,6 @@ export class OpenFile { this.pos = end; if (end > this.size) this.size = end; this.dirty = true; - if (this.size >= STREAM_MIN) this.stream(); } setLength(n: number) { @@ -197,6 +198,18 @@ export class OpenFile { close() { this.release(); } } +// A read-only directory descriptor can be opened, but cannot be read as bytes. +class DirectoryFile extends OpenFile { + constructor(path: string, size: number) { + super(path, new Uint8Array(0), false); + this.size = size; + } + + override read(): number { + throw new PendingThrow("java/io/IOException", "Is a directory"); + } +} + // Reads and writes a character device as a stream. class DeviceFile extends OpenFile { constructor(path: string) { @@ -284,12 +297,17 @@ function notFound(path: string, why = "No such file or directory"): never { } // Opens a path and returns its descriptor number. -export function openFile(t: JThread, path: string, write: boolean, truncate: boolean, append: boolean, readable = true): number { +export function openFile(t: JThread, path: string, write: boolean, truncate: boolean, append: boolean, readable = true, allowDirectory = false): number { const fs = t.vm.fs; const abs = resolvePath(t, path); syncPath(t, abs); const st = fs.stat(abs); - if (st?.dir) notFound(path, "Is a directory"); + if (st?.dir) { + if (!allowDirectory || write) notFound(path, "Is a directory"); + const tab = table(t); + tab.files.set(tab.next, new DirectoryFile(abs, st.size)); + return tab.next++; + } const shared = st ? [...table(t).files.values()].find((o) => o.path === abs && o.constructor === OpenFile)?.contents : undefined; if (st?.device || (st && !shared && !write && st.size >= LAZY_MIN && fs.canReadRange(abs))) { const tab = table(t); @@ -297,7 +315,7 @@ export function openFile(t: JThread, path: string, write: boolean, truncate: boo return tab.next++; } if (write && fs.denied(abs)) notFound(path, "Permission denied"); - if (st && !shared && !truncate && st.size >= STREAM_MIN && fs.canOpenFile(abs)) { + if (st && !shared && !truncate && st.size >= STREAM_MIN && fs.canOpenFile(abs, write)) { const host = openHost(fs, abs, write, readable); if (host) { closeHostsOnExit(t.vm); @@ -330,7 +348,7 @@ export function openFile(t: JThread, path: string, write: boolean, truncate: boo } const f = new OpenFile(abs, data, write, shared); // only a writer can grow a file, so read-only files never hold an opener - if (write && !hostOpeners.has(f.contents) && fs.canOpenFile(abs)) { + if (write && !hostOpeners.has(f.contents) && fs.canOpenFile(abs, true)) { const vm = t.vm; const c = f.contents; hostOpeners.set(c, (current) => { diff --git a/src/natives/nio.ts b/src/natives/nio.ts index 306a16b..9b0c7e1 100644 --- a/src/natives/nio.ts +++ b/src/natives/nio.ts @@ -152,7 +152,7 @@ export const natives: Natives = { if (st?.dir && write) unixError(EISDIR); if ((write || !st) && t.vm.fs.denied(abs)) unixError(EACCES); try { - const fd = openFile(t, abs, write, (flags & O_TRUNC) !== 0, (flags & O_APPEND) !== 0, (flags & O_ACCMODE) !== 1); + const fd = openFile(t, abs, write, (flags & O_TRUNC) !== 0, (flags & O_APPEND) !== 0, (flags & O_ACCMODE) !== 1, true); fileOf(t, fd).ino = fileInode(t, abs); return fd; } catch (e) { @@ -333,7 +333,7 @@ function writeBack(t: JThread, m: Mapping, base: number, addr: number, len: numb if (n <= 0) return true; const save = m.f.pos; m.f.pos = at; - m.f.write(t.vm.mem.u8.slice(addr, addr + n)); + m.f.write(t.vm.mem.u8.slice(addr, addr + n), true); m.f.pos = save; return flush(t, m.f); } diff --git a/test/java/StreamingFile.java b/test/java/StreamingFile.java index 1815829..3c6ea8c 100644 --- a/test/java/StreamingFile.java +++ b/test/java/StreamingFile.java @@ -6,6 +6,15 @@ public class StreamingFile { public static void main(String[] args) throws Exception { + try (var directory = Files.newByteChannel(Path.of(""), java.nio.file.StandardOpenOption.READ, + java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + try { + directory.read(ByteBuffer.allocate(1)); + System.out.println("directory readable"); + } catch (java.io.IOException e) { + System.out.println(e.getClass().getName()); + } + } Path path = Files.createTempFile("streaming-file", ".bin"); try (RandomAccessFile first = new RandomAccessFile(path.toFile(), "rw"); RandomAccessFile second = new RandomAccessFile(path.toFile(), "rw")) { @@ -28,6 +37,29 @@ public static void main(String[] args) throws Exception { } finally { Files.deleteIfExists(path); } + Path sparse = Files.createTempFile("streaming-sparse", ".bin"); + try (RandomAccessFile file = new RandomAccessFile(sparse.toFile(), "rw"); + RandomAccessFile reader = new RandomAccessFile(sparse.toFile(), "r")) { + file.write(17); + file.seek(16L << 20); + file.write(42); + reader.seek(0); + System.out.print(reader.read() + " "); + reader.seek((16L << 20) - 1); + System.out.println(reader.read() + " " + reader.read() + " " + reader.length()); + } finally { + Files.deleteIfExists(sparse); + } + Path append = Files.createTempFile("streaming-append", ".bin"); + try (FileChannel channel = FileChannel.open(append, java.nio.file.StandardOpenOption.WRITE, + java.nio.file.StandardOpenOption.APPEND)) { + channel.write(ByteBuffer.wrap(new byte[] {1, 2})); + channel.position(0); + channel.write(ByteBuffer.wrap(new byte[] {3})); + System.out.println(java.util.Arrays.toString(Files.readAllBytes(append))); + } finally { + Files.deleteIfExists(append); + } Path big = Files.createTempFile("streaming-file", ".bin"); try { try (RandomAccessFile out = new RandomAccessFile(big.toFile(), "rw")) { diff --git a/test/jtreg/passing.txt b/test/jtreg/passing.txt index 91bfd46..80b89cc 100644 --- a/test/jtreg/passing.txt +++ b/test/jtreg/passing.txt @@ -437,6 +437,7 @@ java/io/FileDescriptor/RememberAppend.java java/io/FileDescriptor/Sharing.java java/io/FileDescriptor/Sync.java java/io/FileInputStream/Available.java +java/io/FileInputStream/LargeFileAvailable.java java/io/FileInputStream/NegativeAvailable.java java/io/FileInputStream/OpenDir.java java/io/FileInputStream/OpsAfterClose.java @@ -1923,6 +1924,8 @@ java/nio/channels/Channels/ReadOffset.java java/nio/channels/Channels/ShortWrite.java java/nio/channels/Channels/SocketChannelStreams.java java/nio/channels/Channels/TinyBuffers.java +java/nio/channels/Channels/TransferTo_2GB_transferFrom.java +java/nio/channels/Channels/TransferTo_2GB_transferTo.java java/nio/channels/Channels/Write.java java/nio/channels/ConfigureBlocking.java java/nio/channels/DatagramChannel/AdaptorAsyncCloseAfterReceive.java @@ -1967,6 +1970,7 @@ java/nio/channels/FileChannel/GetClosedChannel.java java/nio/channels/FileChannel/InterruptDeadlock.java java/nio/channels/FileChannel/InterruptMapDeadlock.java java/nio/channels/FileChannel/LongTransferTest.java +java/nio/channels/FileChannel/LoopingTruncate.java java/nio/channels/FileChannel/MapAssertions.java java/nio/channels/FileChannel/MapOverEnd.java java/nio/channels/FileChannel/MapReadOnly.java @@ -1984,8 +1988,11 @@ java/nio/channels/FileChannel/ReleaseOnCloseDeadlock.java java/nio/channels/FileChannel/ScatteringRead.java java/nio/channels/FileChannel/TempDirectBuffersReclamation.java java/nio/channels/FileChannel/Transfer.java +java/nio/channels/FileChannel/Transfer2GPlus.java +java/nio/channels/FileChannel/Transfer4GBFile.java java/nio/channels/FileChannel/TransferFromExtend.java java/nio/channels/FileChannel/TransferOverlappedFileChannel.java +java/nio/channels/FileChannel/TransferTo6GBFile.java java/nio/channels/FileChannel/TransferToAppending.java java/nio/channels/FileChannel/TransferToChannel.java java/nio/channels/FileChannel/TransferToNonWritable.java @@ -2134,6 +2141,7 @@ java/nio/file/Files/Misc.java java/nio/file/Files/Mismatch.java java/nio/file/Files/NameLimits.java java/nio/file/Files/ReadWriteString.java +java/nio/file/Files/SBC.java java/nio/file/Files/StreamTest.java java/nio/file/Files/probeContentType/ForceLoad.java java/nio/file/Files/walkFileTree/MaxDepth.java @@ -3342,10 +3350,12 @@ java/util/zip/ZipEntry/Clone.java java/util/zip/ZipEntry/Constructor.java java/util/zip/ZipEntry/MaxZipEntryFieldSizeTest.java java/util/zip/ZipFile/Assortment.java +java/util/zip/ZipFile/CenSizeTooLarge.java java/util/zip/ZipFile/ClearStaleZipFileInputStreams.java java/util/zip/ZipFile/Comment.java java/util/zip/ZipFile/Constructor.java java/util/zip/ZipFile/CorruptedZipFiles.java +java/util/zip/ZipFile/EndOfCenValidation.java java/util/zip/ZipFile/EnumerateAfterClose.java java/util/zip/ZipFile/FinalizeInflater.java java/util/zip/ZipFile/GetInputStreamNPETest.java @@ -3468,3 +3478,4 @@ javax/xml/jaxp/transform/8062923/XslSubstringTest.java javax/xml/jaxp/validation/8015092/XPathWhiteSpaceTest.java javax/xml/jaxp/validation/8015243/AnyURITest.java javax/xml/jaxp/validation/8049514/FeaturePropagationTest.java +jdk/nio/zipfs/EndOfCenValidation.java diff --git a/test/jtreg/times.tsv b/test/jtreg/times.tsv index 7f1f14e..17038f8 100644 --- a/test/jtreg/times.tsv +++ b/test/jtreg/times.tsv @@ -2,7 +2,9 @@ 108944 java/nio/Buffer/BulkPutBuffer.java 108484 java/net/httpclient/HandshakeFailureTest.java 108005 java/security/cert/CertPathValidator/crlDP/CheckAllCRLs.java +105037 java/nio/channels/Channels/TransferTo_2GB_transferFrom.java 101333 java/net/httpclient/websocket/PendingPingBinaryClose.java +101139 java/nio/channels/Channels/TransferTo_2GB_transferTo.java 100190 java/net/URL/EarlyOrDelayedParsing.java 97238 java/net/httpclient/websocket/PendingPongBinaryClose.java 93525 java/util/zip/DeInflate.java @@ -410,6 +412,7 @@ 11071 java/text/Format/DateFormat/LenientSpaceParsingTest.java 11063 java/beans/XMLEncoder/java_awt_CardLayout.java 11038 java/security/Signature/NONEwithRSA.java +11036 java/nio/channels/FileChannel/Transfer2GPlus.java 11016 java/lang/reflect/Generics/ThreadSafety.java 11011 java/util/concurrent/ConcurrentHashMap/ToArray.java 11003 java/text/Format/DecimalFormat/MantissaDigits.java @@ -913,6 +916,7 @@ 6451 java/util/Optional/BasicLong.java 6449 java/net/httpclient/HttpResponseInputStreamTest.java 6442 java/util/jar/JarFile/mrjar/TestVersionedStream.java +6442 java/util/zip/ZipFile/CenSizeTooLarge.java 6438 java/util/logging/LoggingDeadlock5.java 6438 java/text/Format/DateFormat/Bug4845901.java 6436 java/time/test/java/time/format/TestTextParser.java @@ -2177,11 +2181,13 @@ 3091 javax/xml/jaxp/Encodings/CheckEncodingPropertiesFile.java 3091 java/beans/Introspector/7122138/Test7122138.java 3090 java/lang/management/ClassLoadingMXBean/LoadCounts.java +3090 jdk/nio/zipfs/EndOfCenValidation.java 3089 java/util/logging/FileHandlerPatternExceptions.java 3089 java/net/URLClassLoader/closetest/GetResourceAsStream.java 3087 java/security/SecureRandom/NextBytesNull.java 3087 java/io/LineNumberReader/MarkSplitCRLF.java 3082 java/util/concurrent/ExecutorService/Invoke.java +3082 java/util/zip/ZipFile/EndOfCenValidation.java 3079 java/util/Vector/LastIndexOf.java 3079 java/security/spec/EllipticCurveMatch.java 3073 java/util/jar/JarFile/SignedJarFileGetInputStream.java @@ -3011,6 +3017,7 @@ 2078 java/io/Serializable/InvalidObjectException/TestIoeConstructors.java 2077 java/io/Serializable/enum/array/Test.java 2075 java/lang/Byte/Decode.java +2072 java/nio/channels/FileChannel/TransferTo6GBFile.java 2071 java/util/concurrent/forkjoin/AsyncShutdownNow.java 2070 java/nio/Buffer/Order.java 2070 java/math/BigInteger/CompareToTests.java @@ -3078,6 +3085,7 @@ 2010 java/lang/System/Logger/Level/LoggerLevelTest.java 2010 java/lang/Class/getMethod/NullInParamList.java 2007 java/nio/channels/AsynchronousChannelGroup/Unbounded.java +2006 java/nio/channels/FileChannel/Transfer4GBFile.java 2005 java/lang/Class/getDeclaredField/Exceptions.java 2004 java/util/concurrent/CopyOnWriteArrayList/EqualsRace.java 2000 java/util/logging/Reflect.java @@ -3400,12 +3408,15 @@ 1140 java/security/BasicPermission/ExitVMEquals.java 1133 java/security/BasicPermission/EqualsImplies.java 1128 java/lang/Throwable/LegacyChainedExceptionSerialization.java +1125 java/nio/file/Files/SBC.java 1122 java/nio/channels/FileChannel/ClosedChannelTransfer.java 1112 java/nio/channels/Selector/CloseThenRegister.java 1106 java/nio/channels/FileChannel/Force.java 1100 java/net/URLConnection/GetContentType.java +1077 java/io/FileInputStream/LargeFileAvailable.java 1074 java/nio/channels/Selector/SelectAfterRead.java 1070 java/nio/file/Files/FileAttributes.java +1038 java/nio/channels/FileChannel/LoopingTruncate.java 1036 java/net/URL/Constructor.java 1019 java/nio/channels/DatagramChannel/StressNativeSignal.java 1011 java/nio/channels/SocketChannel/CloseAfterConnect.java diff --git a/test/nio.test.ts b/test/nio.test.ts index 1ad8312..150d0e8 100644 --- a/test/nio.test.ts +++ b/test/nio.test.ts @@ -1,6 +1,6 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, rmSync, truncateSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { MemoryBackend, VFS } from "../src/fs.ts"; @@ -71,3 +71,54 @@ test("NodeBackend retains the errno from a failed chmod", () => { rmSync(dir, { recursive: true, force: true }); } }); + +test("a sparse host file streams before allocating its gap", async () => { + const { openFile, fileOf, closeFd } = await import("../src/natives/io.ts"); + const dir = mkdtempSync(join(tmpdir(), "java-web-sparse-")); + const fs = new VFS(); + let largestWrite = 0; + const backend = new NodeBackend(dir, true); + const open = backend.openFile.bind(backend); + backend.openFile = (...args) => { + const handle = open(...args); + const write = handle.write; + handle.write = (src, pos) => { largestWrite = Math.max(largestWrite, src.length); write(src, pos); }; + return handle; + }; + fs.mount("/", backend); + const t = { vm: { fs, opts: {}, state: new Map(), exited: new Promise(() => {}) } } as unknown as JThread; + try { + const fd = openFile(t, "/file", true, false, false); + const f = fileOf(t, fd); + f.pos = 16 << 20; + f.write(new Uint8Array([42])); + assert.equal(f.data.length, 0); + assert.ok(largestWrite < 1024, `host write was ${largestWrite} bytes`); + assert.equal(f.size, (16 << 20) + 1); + closeFd(t, fd); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + + +test("a large file on a read-only host mount uses a handle", async () => { + const { openFile, fileOf, closeFd } = await import("../src/natives/io.ts"); + const dir = mkdtempSync(join(tmpdir(), "java-web-readonly-")); + const fs = new VFS(); + writeFileSync(join(dir, "file"), new Uint8Array([42])); + truncateSync(join(dir, "file"), 6 << 20); + fs.mount("/", new NodeBackend(dir)); + const t = { vm: { fs, opts: {}, state: new Map(), exited: new Promise(() => {}) } } as unknown as JThread; + try { + const fd = openFile(t, "/file", false, false, false); + const f = fileOf(t, fd); + assert.equal(f.data.length, 0); + const byte = new Uint8Array(1); + assert.equal(f.read(byte, 0, 1), 1); + assert.equal(byte[0], 42); + closeFd(t, fd); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); From 6a1d88bb4867d4f95231fdd69816ea37a1cbcb46 Mon Sep 17 00:00:00 2001 From: Project516 <138796702+Project516@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:53:12 -0600 Subject: [PATCH 2/2] Refuse writes and resizes on a directory descriptor itself --- src/natives/io.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/natives/io.ts b/src/natives/io.ts index 8424323..238e40a 100644 --- a/src/natives/io.ts +++ b/src/natives/io.ts @@ -208,6 +208,14 @@ class DirectoryFile extends OpenFile { override read(): number { throw new PendingThrow("java/io/IOException", "Is a directory"); } + + override write(): void { + throw new PendingThrow("java/io/IOException", "Is a directory"); + } + + override setLength(): void { + throw new PendingThrow("java/io/IOException", "Is a directory"); + } } // Reads and writes a character device as a stream.