Write sparse files without allocating their gaps, so multi-GB file tests pass

src/fs.ts

@@ -297,14 +297,14 @@ export class VFS {
     return data;
   }
 
-  canOpenFile(path: string): boolean {
+  canOpenFile(path: string, write: boolean): boolean {
     const r = this.resolve(path);
-    return !!r && r[0].writable && !!r[0].openFile;
+    return !!r && (!write || r[0].writable) && !!r[0].openFile;
   }
 
   openFile(path: string, write: boolean, truncate: boolean, readable: boolean): FileHandle | undefined {
     const r = this.resolve(path);
-    if (!r || !r[0].writable || !r[0].openFile) return undefined;
+    if (!r || ((write || truncate) && !r[0].writable) || !r[0].openFile) return undefined;
     const h = r[0].openFile(r[1], write, truncate, readable);
     // writes through the handle are changes watchers see, as writes through VFS.write are
     return {

src/natives/io.ts

@@ -54,7 +54,7 @@ const closesHosts = new WeakSet<JVM>();
 
 // A snapshot cannot hold a host handle, nor the opener a file may still need.
 export function streams(f: OpenFile): boolean {
-  return !!f.contents.host || hostOpeners.has(f.contents);
+  return f instanceof DirectoryFile || !!f.contents.host || hostOpeners.has(f.contents);
 }
 
 // Host handles of files the program leaves open close when its VM exits.
@@ -120,17 +120,19 @@ export class OpenFile {
     return n;
   }
 
-  write(src: Uint8Array) {
+  write(src: Uint8Array, atPosition = false) {
+    if (this.append && !atPosition) this.pos = this.size;
     const host = this.contents.host;
     if (host) {
       hostCall(() => host.write(src, this.pos));
       this.pos += src.length;
       if (this.pos > this.size) this.size = this.pos;
       return;
     }
+    const end = this.pos + src.length;
+    if (end >= STREAM_MIN && this.stream()) return this.write(src, atPosition);
     // past the end, the gap reads as zeros (the buffer may hold bytes from before a truncation)
     if (this.pos > this.size) this.data.fill(0, this.size, Math.min(this.pos, this.data.length));
-    const end = this.pos + src.length;
     if (end > this.data.length) {
       const next = new Uint8Array(Math.max(end, this.data.length * 2, 64));
       next.set(this.data.subarray(0, this.size));
@@ -140,7 +142,6 @@ export class OpenFile {
     this.pos = end;
     if (end > this.size) this.size = end;
     this.dirty = true;
-    if (this.size >= STREAM_MIN) this.stream();
   }
 
   setLength(n: number) {
@@ -197,6 +198,26 @@ export class OpenFile {
   close() { this.release(); }
 }
 
+// A read-only directory descriptor can be opened, but cannot be read as bytes.
+class DirectoryFile extends OpenFile {
+  constructor(path: string, size: number) {
+    super(path, new Uint8Array(0), false);
+    this.size = size;
+  }
+
+  override read(): number {
+    throw new PendingThrow("java/io/IOException", "Is a directory");
+  }
+
+  override write(): void {
+    throw new PendingThrow("java/io/IOException", "Is a directory");
+  }
+
+  override setLength(): void {
+    throw new PendingThrow("java/io/IOException", "Is a directory");
+  }
+}
+
 // Reads and writes a character device as a stream.
 class DeviceFile extends OpenFile {
   constructor(path: string) {
@@ -284,20 +305,25 @@ function notFound(path: string, why = "No such file or directory"): never {
 }
 
 // Opens a path and returns its descriptor number.
-export function openFile(t: JThread, path: string, write: boolean, truncate: boolean, append: boolean, readable = true): number {
+export function openFile(t: JThread, path: string, write: boolean, truncate: boolean, append: boolean, readable = true, allowDirectory = false): number {
   const fs = t.vm.fs;
   const abs = resolvePath(t, path);
   syncPath(t, abs);
   const st = fs.stat(abs);
-  if (st?.dir) notFound(path, "Is a directory");
+  if (st?.dir) {
+    if (!allowDirectory || write) notFound(path, "Is a directory");
+    const tab = table(t);
+    tab.files.set(tab.next, new DirectoryFile(abs, st.size));
+    return tab.next++;
+  }
   const shared = st ? [...table(t).files.values()].find((o) => o.path === abs && o.constructor === OpenFile)?.contents : undefined;
   if (st?.device || (st && !shared && !write && st.size >= LAZY_MIN && fs.canReadRange(abs))) {
     const tab = table(t);
     tab.files.set(tab.next, st.device ? new DeviceFile(abs) : new LazyFile(abs, st.size, fs));
     return tab.next++;
   }
   if (write && fs.denied(abs)) notFound(path, "Permission denied");
-  if (st && !shared && !truncate && st.size >= STREAM_MIN && fs.canOpenFile(abs)) {
+  if (st && !shared && !truncate && st.size >= STREAM_MIN && fs.canOpenFile(abs, write)) {
     const host = openHost(fs, abs, write, readable);
     if (host) {
       closeHostsOnExit(t.vm);
@@ -330,7 +356,7 @@ export function openFile(t: JThread, path: string, write: boolean, truncate: boo
   }
   const f = new OpenFile(abs, data, write, shared);
   // only a writer can grow a file, so read-only files never hold an opener
-  if (write && !hostOpeners.has(f.contents) && fs.canOpenFile(abs)) {
+  if (write && !hostOpeners.has(f.contents) && fs.canOpenFile(abs, true)) {
     const vm = t.vm;
     const c = f.contents;
     hostOpeners.set(c, (current) => {

src/natives/nio.ts

@@ -152,7 +152,7 @@ export const natives: Natives = {
     if (st?.dir && write) unixError(EISDIR);
     if ((write || !st) && t.vm.fs.denied(abs)) unixError(EACCES);
     try {
-      const fd = openFile(t, abs, write, (flags & O_TRUNC) !== 0, (flags & O_APPEND) !== 0, (flags & O_ACCMODE) !== 1);
+      const fd = openFile(t, abs, write, (flags & O_TRUNC) !== 0, (flags & O_APPEND) !== 0, (flags & O_ACCMODE) !== 1, true);
       fileOf(t, fd).ino = fileInode(t, abs);
       return fd;
     } catch (e) {
@@ -333,7 +333,7 @@ function writeBack(t: JThread, m: Mapping, base: number, addr: number, len: numb
   if (n <= 0) return true;
   const save = m.f.pos;
   m.f.pos = at;
-  m.f.write(t.vm.mem.u8.slice(addr, addr + n));
+  m.f.write(t.vm.mem.u8.slice(addr, addr + n), true);
   m.f.pos = save;
   return flush(t, m.f);
 }

test/java/StreamingFile.java

@@ -6,6 +6,15 @@
 
 public class StreamingFile {
   public static void main(String[] args) throws Exception {
+    try (var directory = Files.newByteChannel(Path.of(""), java.nio.file.StandardOpenOption.READ,
+                                              java.nio.file.LinkOption.NOFOLLOW_LINKS)) {
+      try {
+        directory.read(ByteBuffer.allocate(1));
+        System.out.println("directory readable");
+      } catch (java.io.IOException e) {
+        System.out.println(e.getClass().getName());
+      }
+    }
     Path path = Files.createTempFile("streaming-file", ".bin");
     try (RandomAccessFile first = new RandomAccessFile(path.toFile(), "rw");
          RandomAccessFile second = new RandomAccessFile(path.toFile(), "rw")) {
@@ -28,6 +37,29 @@ public static void main(String[] args) throws Exception {
     } finally {
       Files.deleteIfExists(path);
     }
+    Path sparse = Files.createTempFile("streaming-sparse", ".bin");
+    try (RandomAccessFile file = new RandomAccessFile(sparse.toFile(), "rw");
+         RandomAccessFile reader = new RandomAccessFile(sparse.toFile(), "r")) {
+      file.write(17);
+      file.seek(16L << 20);
+      file.write(42);
+      reader.seek(0);
+      System.out.print(reader.read() + " ");
+      reader.seek((16L << 20) - 1);
+      System.out.println(reader.read() + " " + reader.read() + " " + reader.length());
+    } finally {
+      Files.deleteIfExists(sparse);
+    }
+    Path append = Files.createTempFile("streaming-append", ".bin");
+    try (FileChannel channel = FileChannel.open(append, java.nio.file.StandardOpenOption.WRITE,
+                                                 java.nio.file.StandardOpenOption.APPEND)) {
+      channel.write(ByteBuffer.wrap(new byte[] {1, 2}));
+      channel.position(0);
+      channel.write(ByteBuffer.wrap(new byte[] {3}));
+      System.out.println(java.util.Arrays.toString(Files.readAllBytes(append)));
+    } finally {
+      Files.deleteIfExists(append);
+    }
     Path big = Files.createTempFile("streaming-file", ".bin");
     try {
       try (RandomAccessFile out = new RandomAccessFile(big.toFile(), "rw")) {

test/jtreg/passing.txt

@@ -3517,3 +3517,14 @@ jdk/nio/zipfs/testng/test/ChannelTests.java
 jdk/nio/zipfs/testng/test/ManifestOrderTest.java
 jdk/nio/zipfs/testng/test/PosixAttributeViewTest.java
 jdk/nio/zipfs/testng/util/ZipFsBaseTest.java
+java/io/FileInputStream/LargeFileAvailable.java
+java/nio/channels/Channels/TransferTo_2GB_transferFrom.java
+java/nio/channels/Channels/TransferTo_2GB_transferTo.java
+java/nio/channels/FileChannel/LoopingTruncate.java
+java/nio/channels/FileChannel/Transfer2GPlus.java
+java/nio/channels/FileChannel/Transfer4GBFile.java
+java/nio/channels/FileChannel/TransferTo6GBFile.java
+java/nio/file/Files/SBC.java
+java/util/zip/ZipFile/CenSizeTooLarge.java
+java/util/zip/ZipFile/EndOfCenValidation.java
+jdk/nio/zipfs/EndOfCenValidation.java

test/jtreg/times.tsv

@@ -4,7 +4,9 @@
 108484	java/net/httpclient/HandshakeFailureTest.java
 108005	java/security/cert/CertPathValidator/crlDP/CheckAllCRLs.java
 105184	java/util/zip/ZipFile/ZipFileSharedSourceTest.java
+105037	java/nio/channels/Channels/TransferTo_2GB_transferFrom.java
 101333	java/net/httpclient/websocket/PendingPingBinaryClose.java
+101139	java/nio/channels/Channels/TransferTo_2GB_transferTo.java
 100190	java/net/URL/EarlyOrDelayedParsing.java
 97238	java/net/httpclient/websocket/PendingPongBinaryClose.java
 93525	java/util/zip/DeInflate.java
@@ -424,6 +426,7 @@
 11071	java/text/Format/DateFormat/LenientSpaceParsingTest.java
 11063	java/beans/XMLEncoder/java_awt_CardLayout.java
 11038	java/security/Signature/NONEwithRSA.java
+11036	java/nio/channels/FileChannel/Transfer2GPlus.java
 11019	jdk/nio/zipfs/testng/test/ManifestOrderTest.java
 11016	java/lang/reflect/Generics/ThreadSafety.java
 11011	java/util/concurrent/ConcurrentHashMap/ToArray.java
@@ -941,6 +944,7 @@
 6451	java/util/Optional/BasicLong.java
 6449	java/net/httpclient/HttpResponseInputStreamTest.java
 6442	java/util/jar/JarFile/mrjar/TestVersionedStream.java
+6442	java/util/zip/ZipFile/CenSizeTooLarge.java
 6438	java/util/logging/LoggingDeadlock5.java
 6438	java/text/Format/DateFormat/Bug4845901.java
 6436	java/time/test/java/time/format/TestTextParser.java
@@ -2226,11 +2230,13 @@
 3091	javax/xml/jaxp/Encodings/CheckEncodingPropertiesFile.java
 3091	java/beans/Introspector/7122138/Test7122138.java
 3090	java/lang/management/ClassLoadingMXBean/LoadCounts.java
+3090	jdk/nio/zipfs/EndOfCenValidation.java
 3089	java/util/logging/FileHandlerPatternExceptions.java
 3089	java/net/URLClassLoader/closetest/GetResourceAsStream.java
 3087	java/security/SecureRandom/NextBytesNull.java
 3087	java/io/LineNumberReader/MarkSplitCRLF.java
 3082	java/util/concurrent/ExecutorService/Invoke.java
+3082	java/util/zip/ZipFile/EndOfCenValidation.java
 3079	java/util/Vector/LastIndexOf.java
 3079	java/security/spec/EllipticCurveMatch.java
 3073	java/util/jar/JarFile/SignedJarFileGetInputStream.java
@@ -3060,6 +3066,7 @@
 2078	java/io/Serializable/InvalidObjectException/TestIoeConstructors.java
 2077	java/io/Serializable/enum/array/Test.java
 2075	java/lang/Byte/Decode.java
+2072	java/nio/channels/FileChannel/TransferTo6GBFile.java
 2071	java/util/concurrent/forkjoin/AsyncShutdownNow.java
 2070	java/nio/Buffer/Order.java
 2070	java/math/BigInteger/CompareToTests.java
@@ -3127,6 +3134,7 @@
 2010	java/lang/System/Logger/Level/LoggerLevelTest.java
 2010	java/lang/Class/getMethod/NullInParamList.java
 2007	java/nio/channels/AsynchronousChannelGroup/Unbounded.java
+2006	java/nio/channels/FileChannel/Transfer4GBFile.java
 2005	java/lang/Class/getDeclaredField/Exceptions.java
 2004	java/util/concurrent/CopyOnWriteArrayList/EqualsRace.java
 2000	java/util/logging/Reflect.java
@@ -3449,12 +3457,15 @@
 1140	java/security/BasicPermission/ExitVMEquals.java
 1133	java/security/BasicPermission/EqualsImplies.java
 1128	java/lang/Throwable/LegacyChainedExceptionSerialization.java
+1125	java/nio/file/Files/SBC.java
 1122	java/nio/channels/FileChannel/ClosedChannelTransfer.java
 1112	java/nio/channels/Selector/CloseThenRegister.java
 1106	java/nio/channels/FileChannel/Force.java
 1100	java/net/URLConnection/GetContentType.java
+1077	java/io/FileInputStream/LargeFileAvailable.java
 1074	java/nio/channels/Selector/SelectAfterRead.java
 1070	java/nio/file/Files/FileAttributes.java
+1038	java/nio/channels/FileChannel/LoopingTruncate.java
 1036	java/net/URL/Constructor.java
 1019	java/nio/channels/DatagramChannel/StressNativeSignal.java
 1011	java/nio/channels/SocketChannel/CloseAfterConnect.java

test/nio.test.ts

@@ -1,6 +1,6 @@
 import { test } from "node:test";
 import assert from "node:assert/strict";
-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { mkdtempSync, rmSync, truncateSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { MemoryBackend, VFS } from "../src/fs.ts";
@@ -71,3 +71,54 @@ test("NodeBackend retains the errno from a failed chmod", () => {
     rmSync(dir, { recursive: true, force: true });
   }
 });
+
+test("a sparse host file streams before allocating its gap", async () => {
+  const { openFile, fileOf, closeFd } = await import("../src/natives/io.ts");
+  const dir = mkdtempSync(join(tmpdir(), "java-web-sparse-"));
+  const fs = new VFS();
+  let largestWrite = 0;
+  const backend = new NodeBackend(dir, true);
+  const open = backend.openFile.bind(backend);
+  backend.openFile = (...args) => {
+    const handle = open(...args);
+    const write = handle.write;
+    handle.write = (src, pos) => { largestWrite = Math.max(largestWrite, src.length); write(src, pos); };
+    return handle;
+  };
+  fs.mount("/", backend);
+  const t = { vm: { fs, opts: {}, state: new Map(), exited: new Promise(() => {}) } } as unknown as JThread;
+  try {
+    const fd = openFile(t, "/file", true, false, false);
+    const f = fileOf(t, fd);
+    f.pos = 16 << 20;
+    f.write(new Uint8Array([42]));
+    assert.equal(f.data.length, 0);
+    assert.ok(largestWrite < 1024, `host write was ${largestWrite} bytes`);
+    assert.equal(f.size, (16 << 20) + 1);
+    closeFd(t, fd);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+
+test("a large file on a read-only host mount uses a handle", async () => {
+  const { openFile, fileOf, closeFd } = await import("../src/natives/io.ts");
+  const dir = mkdtempSync(join(tmpdir(), "java-web-readonly-"));
+  const fs = new VFS();
+  writeFileSync(join(dir, "file"), new Uint8Array([42]));
+  truncateSync(join(dir, "file"), 6 << 20);
+  fs.mount("/", new NodeBackend(dir));
+  const t = { vm: { fs, opts: {}, state: new Map(), exited: new Promise(() => {}) } } as unknown as JThread;
+  try {
+    const fd = openFile(t, "/file", false, false, false);
+    const f = fileOf(t, fd);
+    assert.equal(f.data.length, 0);
+    const byte = new Uint8Array(1);
+    assert.equal(f.read(byte, 0, 1), 1);
+    assert.equal(byte[0], 42);
+    closeFd(t, fd);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});