Parse the boot index of each module the first time a path in it is read

scripts/build-web.ts

@@ -129,6 +129,8 @@ const inJavacPack = new Set(javacFiles);
 // says where each file is, so the worker can stat and list offline. Files of a
 // package sit together, so one block read serves several classes.
 const lines: string[] = [];
+// a module's file lines, which a start parses only once it reads that module
+const moduleLines = new Map<string, string[]>();
 // jdk.localedata's lines go in jdk/jdk.localedata.txt, which a page fetches only with localeData on
 const localeLines: string[] = [];
 const images = new Map<string, { parts: Uint8Array[]; size: number }>();
@@ -147,7 +149,13 @@ const walk = (dir: string) => {
       const name = imageOf(rel);
       let image = images.get(name);
       if (!image) images.set(name, (image = { parts: [], size: 0 }));
-      out.push(`f ${rel} ${st.size} ${name}:${image.size}${inJavacPack.has(rel) ? " javac" : ""}`);
+      const module = out === lines ? /^\/modules\/([^/]+)\//.exec(rel)?.[1] : undefined;
+      let fileLines = out;
+      if (module) {
+        fileLines = moduleLines.get(module) ?? [];
+        moduleLines.set(module, fileLines);
+      }
+      fileLines.push(`f ${rel} ${st.size} ${name}:${image.size}${inJavacPack.has(rel) ? " javac" : ""}`);
       image.parts.push(readFileSync(p));
       image.size += st.size;
     }
@@ -181,18 +189,21 @@ function writePack(file: string, entries: [string, Uint8Array][]) {
   return `${(size / 1024).toFixed(0)} KB, ${entries.length} files`;
 }
 
-let previous = "";
-const compactIndex = lines.map((line) => {
-  let n = 0;
-  while (n < line.length && line[n] === previous[n]) n++;
-  // Keep surrogate pairs together when encoding the suffix as UTF-8.
-  if (n > 0 && line.charCodeAt(n - 1) >= 0xd800 && line.charCodeAt(n - 1) <= 0xdbff) n--;
-  const result = `${n} ${line.slice(n)}`;
-  previous = line;
-  return result;
-}).join("\n");
+// Each line as the length it shares with the line before, then the rest.
+function prefixCode(lines: string[]) {
+  let previous = "";
+  return enc.encode(lines.map((line) => {
+    let n = 0;
+    while (n < line.length && line[n] === previous[n]) n++;
+    // Keep surrogate pairs together when encoding the suffix as UTF-8.
+    if (n > 0 && line.charCodeAt(n - 1) >= 0xd800 && line.charCodeAt(n - 1) <= 0xdbff) n--;
+    previous = line;
+    return `${n} ${line.slice(n)}`;
+  }).join("\n"));
+}
 const boot = writePack(join(dist, "boot.pack"), [
-  ["/index.prefix", enc.encode(compactIndex)],
+  ["/index.prefix", prefixCode(lines)],
+  ...[...moduleLines].map(([m, l]): [string, Uint8Array] => [`/index/${m}.prefix`, prefixCode(l)]),
   ...[...recorder.seen].sort().map((p): [string, Uint8Array] => [p, recorder.read(p)!]),
 ]);
 const javacPack = writePack(join(dist, "jdk", "javac.pack"), javacFiles.map((p) => [p, javacRecorder.read(p)!]));

src/browser/http.ts

@@ -7,7 +7,7 @@ import type { Backend, Stat } from "../fs.ts";
 export interface Pack {
   files: Map<string, Uint8Array>;
   // path -> size, or -1 for directories
-  index: Map<string, number> | null;
+  index: PackIndex | null;
   // path -> name of the lazily fetched pack that holds it
   lazy: Map<string, string>;
   // path -> the image file holding it and its offset there
@@ -37,33 +37,91 @@ export function parsePack(buf: ArrayBuffer): Pack {
   const pack: Pack = { files, index: null, lazy: new Map(), offsets: new Map() };
   const raw = files.get("/index.prefix") ?? files.get("/index.txt");
   if (raw) {
-    pack.index = new Map([["/", -1]]);
+    pack.index = new PackIndex(pack);
     addIndex(pack, dec.decode(raw), files.has("/index.prefix"));
     files.delete("/index.txt");
     files.delete("/index.prefix");
+    for (const [p, data] of files) {
+      if (!p.startsWith("/index/")) continue;
+      pack.index.modules.set(p.slice(7, -7), data);
+      files.delete(p);
+    }
   }
   return pack;
 }
 
-// Adds index lines to a pack that has an index.
+// A pack's index. A module's file lines ("/index/<module>.prefix") are parsed
+// the first time a path in that module is looked up, since a start reads few
+// of the image's modules.
+export class PackIndex extends Map<string, number> {
+  readonly modules = new Map<string, Uint8Array>();
+  private readonly pack: Pack;
+
+  constructor(pack: Pack) {
+    super([["/", -1]]);
+    this.pack = pack;
+  }
+
+  override get(path: string) {
+    this.load(path);
+    return super.get(path);
+  }
+
+  override has(path: string) {
+    this.load(path);
+    return super.has(path);
+  }
+
+  // Parses the lines of the module path is in. The module directories
+  // themselves are in the main index.
+  load(path: string) {
+    if (this.modules.size === 0 || !path.startsWith("/modules/")) return;
+    const end = path.indexOf("/", 9);
+    if (end < 0) return;
+    const module = path.slice(9, end);
+    const raw = this.modules.get(module);
+    if (!raw) return;
+    this.modules.delete(module);
+    addIndex(this.pack, new TextDecoder().decode(raw), true);
+  }
+}
+
+// Adds index lines to a pack that has an index. Runs on every start, so it
+// scans with indexOf rather than split and regexes.
 export function addIndex(pack: Pack, text: string, prefix = false) {
+  const index = pack.index!;
+  const images = new Map<string, string>();
   let previous = "";
-  for (let line of text.split("\n")) {
+  for (let s = 0; s < text.length;) {
+    let e = text.indexOf("\n", s);
+    if (e < 0) e = text.length;
+    let line = text.slice(s, e);
+    s = e + 1;
     if (prefix) {
       const space = line.indexOf(" ");
-      line = previous.slice(0, Number(line.slice(0, space))) + line.slice(space + 1);
+      line = previous.slice(0, +line.slice(0, space)) + line.slice(space + 1);
       previous = line;
     }
-    const [kind, path, size, ...rest] = line.split(" ");
-    if (kind === "d") pack.index!.set(path, -1);
-    else if (kind === "f") {
-      pack.index!.set(path, Number(size));
-      const at = rest[0] === undefined ? null : /^([^:]+):(\d+)$/.exec(rest[0]);
-      if (at) {
-        pack.offsets.set(path, { image: `/${at[1]}.jar`, at: Number(at[2]) });
-        rest.shift();
+    const a = line.indexOf(" ", 2);
+    const path = a < 0 ? line.slice(2) : line.slice(2, a);
+    if (line[0] === "d") index.set(path, -1);
+    else if (line[0] === "f") {
+      let b = line.indexOf(" ", a + 1);
+      if (b < 0) b = line.length;
+      index.set(path, +line.slice(a + 1, b));
+      if (b === line.length) continue;
+      let c = line.indexOf(" ", b + 1);
+      if (c < 0) c = line.length;
+      const colon = line.lastIndexOf(":", c);
+      if (colon > b) {
+        const module = line.slice(b + 1, colon);
+        let image = images.get(module);
+        if (image === undefined) images.set(module, (image = `/${module}.jar`));
+        pack.offsets.set(path, { image, at: +line.slice(colon + 1, c) });
+        if (c === line.length) continue;
+        b = c;
       }
-      if (rest[0]) pack.lazy.set(path, rest[0]);
+      pack.lazy.set(path, line.slice(b + 1));
     }
   }
 }
@@ -151,6 +209,8 @@ export class HttpBackend implements Backend {
   read(path: string): Uint8Array | null {
     const packed = this.pack.files.get(path);
     if (packed) return packed;
+    // before lazy: a module's lazy entries appear once its index is parsed
+    if (this.pack.index && !this.pack.index.has(path)) return null;
     const name = this.pack.lazy.get(path);
     if (name !== undefined && !this.fetchedPacks.has(name)) {
       this.fetchedPacks.add(name);
@@ -159,7 +219,6 @@ export class HttpBackend implements Backend {
       return this.read(path);
     }
     if (this.cache.has(path)) return this.cache.get(path)!;
-    if (this.pack.index && !this.pack.index.has(path)) return null;
     const at = this.pack.offsets.get(path);
     if (at !== undefined) return this.readRange(path, 0, this.pack.index!.get(path)!);
     const xhr = this.request("GET", path);
@@ -174,7 +233,8 @@ export class HttpBackend implements Backend {
     const whole = this.pack.files.get(path) ?? this.cache.get(path);
     if (whole) return whole.subarray(off, off + len);
     if (len === 0) return new Uint8Array(0);
-    const at = this.pack.offsets.get(path);
+    // has() parses the path's module first; image files themselves are not indexed
+    const at = this.pack.index?.has(path) ? this.pack.offsets.get(path) : undefined;
     if (at !== undefined) return this.readRange(at.image, at.at + off, len);
     let blocks = this.blocks.get(path);
     if (!blocks) this.blocks.set(path, (blocks = new Map()));
@@ -233,9 +293,10 @@ export class HttpBackend implements Backend {
 }
 
 // The names in a directory of a pack index, or null if path is not one.
-export function listIndex(idx: Map<string, number>, path: string): string[] | null {
+export function listIndex(idx: PackIndex, path: string): string[] | null {
   if (idx.get(path) !== -1) return null;
   const prefix = path === "/" ? "/" : path + "/";
+  idx.load(prefix);
   const out: string[] = [];
   for (const p of idx.keys()) {
     if (p.startsWith(prefix) && p.length > prefix.length && !p.slice(prefix.length).includes("/")) out.push(p.slice(prefix.length));

test/image.test.ts

@@ -74,6 +74,24 @@ test("compact index keeps paths, sizes, offsets, and directory order", () => {
   assert.equal(image.lazy.get("/lib/two"), "javac");
 });
 
+test("a module's index lines are parsed when a path in it is first looked up", () => {
+  const buf = pack([
+    ["/index.prefix", "0 d /modules\n0 d /modules/a\n0 d /modules/a/p\n0 d /modules/b"],
+    ["/index/a.prefix", "0 f /modules/a/p/X.class 5 a:0\n15 Y.class 6 a:5 javac"],
+    ["/index/b.prefix", "0 f /modules/b/Z.class 7 b:0"],
+  ]);
+  const image = parsePack(buf.buffer as ArrayBuffer);
+  assert.equal(image.files.size, 0);
+  assert.equal(image.offsets.size, 0);
+  assert.deepEqual(listIndex(image.index!, "/modules/a/p"), ["X.class", "Y.class"]);
+  assert.equal(image.index!.get("/modules/a/p/Y.class"), 6);
+  assert.deepEqual(image.offsets.get("/modules/a/p/Y.class"), { image: "/a.jar", at: 5 });
+  assert.equal(image.lazy.get("/modules/a/p/Y.class"), "javac");
+  assert.equal(image.offsets.has("/modules/b/Z.class"), false);
+  assert.equal(image.index!.has("/modules/b/Z.class"), true);
+  assert.deepEqual(listIndex(image.index!, "/modules"), ["a", "b"]);
+});
+
 const node = join(root, "dist", "node.js");
 const skip = existsSync(node) ? false : "run `pnpm build` first";
 const out = join(root, "build", "test-image");