From 14700a86229354d07658e4b5a478db859f3ef259 Mon Sep 17 00:00:00 2001 From: Project516 <138796702+Project516@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:18:39 -0600 Subject: [PATCH 1/4] Parse the boot index of each module the first time a path in it is read --- scripts/build-web.ts | 35 ++++++++++------ src/browser/http.ts | 95 ++++++++++++++++++++++++++++++++++++-------- test/image.test.ts | 18 +++++++++ 3 files changed, 119 insertions(+), 29 deletions(-) diff --git a/scripts/build-web.ts b/scripts/build-web.ts index 9808525..d9f42ed 100644 --- a/scripts/build-web.ts +++ b/scripts/build-web.ts @@ -129,6 +129,8 @@ const inJavacPack = new Set(javacFiles); // says where each file is, so the worker can stat and list offline. Files of a // package sit together, so one block read serves several classes. const lines: string[] = []; +// a module's file lines, which a start parses only once it reads that module +const moduleLines = new Map(); // jdk.localedata's lines go in jdk/jdk.localedata.txt, which a page fetches only with localeData on const localeLines: string[] = []; const images = new Map(); @@ -147,7 +149,13 @@ const walk = (dir: string) => { const name = imageOf(rel); let image = images.get(name); if (!image) images.set(name, (image = { parts: [], size: 0 })); - out.push(`f ${rel} ${st.size} ${name}:${image.size}${inJavacPack.has(rel) ? " javac" : ""}`); + const module = out === lines ? /^\/modules\/([^/]+)\//.exec(rel)?.[1] : undefined; + let fileLines = out; + if (module) { + fileLines = moduleLines.get(module) ?? []; + moduleLines.set(module, fileLines); + } + fileLines.push(`f ${rel} ${st.size} ${name}:${image.size}${inJavacPack.has(rel) ? " javac" : ""}`); image.parts.push(readFileSync(p)); image.size += st.size; } @@ -181,18 +189,21 @@ function writePack(file: string, entries: [string, Uint8Array][]) { return `${(size / 1024).toFixed(0)} KB, ${entries.length} files`; } -let previous = ""; -const compactIndex = lines.map((line) => { - let n = 0; - while (n < line.length && line[n] === previous[n]) n++; - // Keep surrogate pairs together when encoding the suffix as UTF-8. - if (n > 0 && line.charCodeAt(n - 1) >= 0xd800 && line.charCodeAt(n - 1) <= 0xdbff) n--; - const result = `${n} ${line.slice(n)}`; - previous = line; - return result; -}).join("\n"); +// Each line as the length it shares with the line before, then the rest. +function prefixCode(lines: string[]) { + let previous = ""; + return enc.encode(lines.map((line) => { + let n = 0; + while (n < line.length && line[n] === previous[n]) n++; + // Keep surrogate pairs together when encoding the suffix as UTF-8. + if (n > 0 && line.charCodeAt(n - 1) >= 0xd800 && line.charCodeAt(n - 1) <= 0xdbff) n--; + previous = line; + return `${n} ${line.slice(n)}`; + }).join("\n")); +} const boot = writePack(join(dist, "boot.pack"), [ - ["/index.prefix", enc.encode(compactIndex)], + ["/index.prefix", prefixCode(lines)], + ...[...moduleLines].map(([m, l]): [string, Uint8Array] => [`/index/${m}.prefix`, prefixCode(l)]), ...[...recorder.seen].sort().map((p): [string, Uint8Array] => [p, recorder.read(p)!]), ]); const javacPack = writePack(join(dist, "jdk", "javac.pack"), javacFiles.map((p) => [p, javacRecorder.read(p)!])); diff --git a/src/browser/http.ts b/src/browser/http.ts index 1b7d355..5e586d5 100644 --- a/src/browser/http.ts +++ b/src/browser/http.ts @@ -7,7 +7,7 @@ import type { Backend, Stat } from "../fs.ts"; export interface Pack { files: Map; // path -> size, or -1 for directories - index: Map | null; + index: PackIndex | null; // path -> name of the lazily fetched pack that holds it lazy: Map; // path -> the image file holding it and its offset there @@ -37,33 +37,91 @@ export function parsePack(buf: ArrayBuffer): Pack { const pack: Pack = { files, index: null, lazy: new Map(), offsets: new Map() }; const raw = files.get("/index.prefix") ?? files.get("/index.txt"); if (raw) { - pack.index = new Map([["/", -1]]); + pack.index = new PackIndex(pack); addIndex(pack, dec.decode(raw), files.has("/index.prefix")); files.delete("/index.txt"); files.delete("/index.prefix"); + for (const [p, data] of files) { + if (!p.startsWith("/index/")) continue; + pack.index.modules.set(p.slice(7, -7), data); + files.delete(p); + } } return pack; } -// Adds index lines to a pack that has an index. +// A pack's index. A module's file lines ("/index/.prefix") are parsed +// the first time a path in that module is looked up, since a start reads few +// of the image's modules. +export class PackIndex extends Map { + readonly modules = new Map(); + private readonly pack: Pack; + + constructor(pack: Pack) { + super([["/", -1]]); + this.pack = pack; + } + + override get(path: string) { + this.load(path); + return super.get(path); + } + + override has(path: string) { + this.load(path); + return super.has(path); + } + + // Parses the lines of the module path is in. The module directories + // themselves are in the main index. + load(path: string) { + if (this.modules.size === 0 || !path.startsWith("/modules/")) return; + const end = path.indexOf("/", 9); + if (end < 0) return; + const module = path.slice(9, end); + const raw = this.modules.get(module); + if (!raw) return; + this.modules.delete(module); + addIndex(this.pack, new TextDecoder().decode(raw), true); + } +} + +// Adds index lines to a pack that has an index. Runs on every start, so it +// scans with indexOf rather than split and regexes. export function addIndex(pack: Pack, text: string, prefix = false) { + const index = pack.index!; + const images = new Map(); let previous = ""; - for (let line of text.split("\n")) { + for (let s = 0; s < text.length;) { + let e = text.indexOf("\n", s); + if (e < 0) e = text.length; + let line = text.slice(s, e); + s = e + 1; if (prefix) { const space = line.indexOf(" "); - line = previous.slice(0, Number(line.slice(0, space))) + line.slice(space + 1); + line = previous.slice(0, +line.slice(0, space)) + line.slice(space + 1); previous = line; } - const [kind, path, size, ...rest] = line.split(" "); - if (kind === "d") pack.index!.set(path, -1); - else if (kind === "f") { - pack.index!.set(path, Number(size)); - const at = rest[0] === undefined ? null : /^([^:]+):(\d+)$/.exec(rest[0]); - if (at) { - pack.offsets.set(path, { image: `/${at[1]}.jar`, at: Number(at[2]) }); - rest.shift(); + const a = line.indexOf(" ", 2); + const path = a < 0 ? line.slice(2) : line.slice(2, a); + if (line[0] === "d") index.set(path, -1); + else if (line[0] === "f") { + let b = line.indexOf(" ", a + 1); + if (b < 0) b = line.length; + index.set(path, +line.slice(a + 1, b)); + if (b === line.length) continue; + let c = line.indexOf(" ", b + 1); + if (c < 0) c = line.length; + const colon = line.lastIndexOf(":", c); + if (colon > b) { + const module = line.slice(b + 1, colon); + let image = images.get(module); + if (image === undefined) images.set(module, (image = `/${module}.jar`)); + pack.offsets.set(path, { image, at: +line.slice(colon + 1, c) }); + if (c === line.length) continue; + b = c; } - if (rest[0]) pack.lazy.set(path, rest[0]); + pack.lazy.set(path, line.slice(b + 1)); } } } @@ -151,6 +209,8 @@ export class HttpBackend implements Backend { read(path: string): Uint8Array | null { const packed = this.pack.files.get(path); if (packed) return packed; + // before lazy: a module's lazy entries appear once its index is parsed + if (this.pack.index && !this.pack.index.has(path)) return null; const name = this.pack.lazy.get(path); if (name !== undefined && !this.fetchedPacks.has(name)) { this.fetchedPacks.add(name); @@ -159,7 +219,6 @@ export class HttpBackend implements Backend { return this.read(path); } if (this.cache.has(path)) return this.cache.get(path)!; - if (this.pack.index && !this.pack.index.has(path)) return null; const at = this.pack.offsets.get(path); if (at !== undefined) return this.readRange(path, 0, this.pack.index!.get(path)!); const xhr = this.request("GET", path); @@ -174,7 +233,8 @@ export class HttpBackend implements Backend { const whole = this.pack.files.get(path) ?? this.cache.get(path); if (whole) return whole.subarray(off, off + len); if (len === 0) return new Uint8Array(0); - const at = this.pack.offsets.get(path); + // has() parses the path's module first; image files themselves are not indexed + const at = this.pack.index?.has(path) ? this.pack.offsets.get(path) : undefined; if (at !== undefined) return this.readRange(at.image, at.at + off, len); let blocks = this.blocks.get(path); if (!blocks) this.blocks.set(path, (blocks = new Map())); @@ -233,9 +293,10 @@ export class HttpBackend implements Backend { } // The names in a directory of a pack index, or null if path is not one. -export function listIndex(idx: Map, path: string): string[] | null { +export function listIndex(idx: PackIndex, path: string): string[] | null { if (idx.get(path) !== -1) return null; const prefix = path === "/" ? "/" : path + "/"; + idx.load(prefix); const out: string[] = []; for (const p of idx.keys()) { if (p.startsWith(prefix) && p.length > prefix.length && !p.slice(prefix.length).includes("/")) out.push(p.slice(prefix.length)); diff --git a/test/image.test.ts b/test/image.test.ts index 18fea82..886142f 100644 --- a/test/image.test.ts +++ b/test/image.test.ts @@ -74,6 +74,24 @@ test("compact index keeps paths, sizes, offsets, and directory order", () => { assert.equal(image.lazy.get("/lib/two"), "javac"); }); +test("a module's index lines are parsed when a path in it is first looked up", () => { + const buf = pack([ + ["/index.prefix", "0 d /modules\n0 d /modules/a\n0 d /modules/a/p\n0 d /modules/b"], + ["/index/a.prefix", "0 f /modules/a/p/X.class 5 a:0\n15 Y.class 6 a:5 javac"], + ["/index/b.prefix", "0 f /modules/b/Z.class 7 b:0"], + ]); + const image = parsePack(buf.buffer as ArrayBuffer); + assert.equal(image.files.size, 0); + assert.equal(image.offsets.size, 0); + assert.deepEqual(listIndex(image.index!, "/modules/a/p"), ["X.class", "Y.class"]); + assert.equal(image.index!.get("/modules/a/p/Y.class"), 6); + assert.deepEqual(image.offsets.get("/modules/a/p/Y.class"), { image: "/a.jar", at: 5 }); + assert.equal(image.lazy.get("/modules/a/p/Y.class"), "javac"); + assert.equal(image.offsets.has("/modules/b/Z.class"), false); + assert.equal(image.index!.has("/modules/b/Z.class"), true); + assert.deepEqual(listIndex(image.index!, "/modules"), ["a", "b"]); +}); + const node = join(root, "dist", "node.js"); const skip = existsSync(node) ? false : "run `pnpm build` first"; const out = join(root, "build", "test-image"); From 41f1d61997eadae659d16eab22cc1dff2b917877 Mon Sep 17 00:00:00 2001 From: Project516 <138796702+Project516@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:50:09 -0600 Subject: [PATCH 2/4] List pack index directories from a children map built once instead of scanning every path --- src/browser/http.ts | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/browser/http.ts b/src/browser/http.ts index 5e586d5..62c9c24 100644 --- a/src/browser/http.ts +++ b/src/browser/http.ts @@ -56,6 +56,9 @@ export function parsePack(buf: ArrayBuffer): Pack { export class PackIndex extends Map { readonly modules = new Map(); private readonly pack: Pack; + // directory -> names in it, built for the entries there were at the time + private dirs: Map | null = null; + private dirsSize = 0; constructor(pack: Pack) { super([["/", -1]]); @@ -72,6 +75,24 @@ export class PackIndex extends Map { return super.has(path); } + // The names in each directory. Entries are only ever added, so a change in + // size means new ones. + children(): Map { + if (this.dirs && this.dirsSize === this.size) return this.dirs; + const dirs = new Map(); + for (const p of this.keys()) { + if (p === "/") continue; + const i = p.lastIndexOf("/"); + const parent = i === 0 ? "/" : p.slice(0, i); + let names = dirs.get(parent); + if (!names) dirs.set(parent, (names = [])); + names.push(p.slice(i + 1)); + } + this.dirs = dirs; + this.dirsSize = this.size; + return dirs; + } + // Parses the lines of the module path is in. The module directories // themselves are in the main index. load(path: string) { @@ -297,9 +318,5 @@ export function listIndex(idx: PackIndex, path: string): string[] | null { if (idx.get(path) !== -1) return null; const prefix = path === "/" ? "/" : path + "/"; idx.load(prefix); - const out: string[] = []; - for (const p of idx.keys()) { - if (p.startsWith(prefix) && p.length > prefix.length && !p.slice(prefix.length).includes("/")) out.push(p.slice(prefix.length)); - } - return out; + return [...(idx.children().get(path) ?? [])]; } From 1a4a5d432e89d805572a5b05eb775fd7821c629c Mon Sep 17 00:00:00 2001 From: Project516 <138796702+Project516@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:53:47 -0600 Subject: [PATCH 3/4] Look up native field slots without building a key string --- src/jvm.ts | 15 ++++++++------- src/natives/nio.ts | 6 ++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/jvm.ts b/src/jvm.ts index add6487..6b2f6c3 100644 --- a/src/jvm.ts +++ b/src/jvm.ts @@ -117,7 +117,8 @@ export class JVM { private natives = new Map(); private interned = new Map(); readonly prims = new Map(); - private slots = new Map(); + // owner -> field name -> slot; two lookups of literal strings beat building a key + private slots = new Map>(); readonly threads = new Set(); private runQueue: JThread[] = []; private queued = new Set(); @@ -716,14 +717,14 @@ export class JVM { // ---- fields by name ------------------------------------------------------ slotOf(owner: string, name: string): number { - const key = owner + "." + name; - let s = this.slots.get(key); + let byName = this.slots.get(owner); + if (!byName) this.slots.set(owner, (byName = new Map())); + let s = byName.get(name); if (s === undefined) { - const c = this.loadBootOrFail(owner); - const f = c.findField(name); - if (!f) throw new Error(`no field ${key}`); + const f = this.loadBootOrFail(owner).findField(name); + if (!f) throw new Error(`no field ${owner}.${name}`); s = f.slot; - this.slots.set(key, s); + byName.set(name, s); } return s; } diff --git a/src/natives/nio.ts b/src/natives/nio.ts index 5771d09..df7336c 100644 --- a/src/natives/nio.ts +++ b/src/natives/nio.ts @@ -88,10 +88,8 @@ function fillStat(t: JThread, attrs: JObject, st: Stat, writable: boolean, ino: vm.setField(attrs, A, "st_uid", 1000); vm.setField(attrs, A, "st_gid", 1000); vm.setField(attrs, A, "st_size", BigInt(size)); - for (const k of ["atime", "mtime", "ctime"]) { - vm.setField(attrs, A, `st_${k}_sec`, sec); - vm.setField(attrs, A, `st_${k}_nsec`, nsec); - } + for (const k of ["st_atime_sec", "st_mtime_sec", "st_ctime_sec"]) vm.setField(attrs, A, k, sec); + for (const k of ["st_atime_nsec", "st_mtime_nsec", "st_ctime_nsec"]) vm.setField(attrs, A, k, nsec); } // open mount table handles: whether the one entry has been read From 5f9bb231dbfcb72b27e1c32e9cf3f9e7daf76e6d Mon Sep 17 00:00:00 2001 From: Project516 <138796702+Project516@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:12:26 -0600 Subject: [PATCH 4/4] Share one TextDecoder for pack indexes --- src/browser/http.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/browser/http.ts b/src/browser/http.ts index 62c9c24..0ca8baa 100644 --- a/src/browser/http.ts +++ b/src/browser/http.ts @@ -14,6 +14,8 @@ export interface Pack { offsets: Map; } +const utf8 = new TextDecoder(); + // Pack layout: repeated [u16 path length][path][u32 size][bytes]. In the boot // pack, "/index.prefix" holds prefix-coded "d path" and "f path size // [image:offset] [pack]" lines. A file with an offset is read in ranges from @@ -24,11 +26,10 @@ export interface Pack { export function parsePack(buf: ArrayBuffer): Pack { const dv = new DataView(buf); const u8 = new Uint8Array(buf); - const dec = new TextDecoder(); const files = new Map(); for (let p = 0; p < buf.byteLength;) { const n = dv.getUint16(p); - const path = dec.decode(u8.subarray(p + 2, p + 2 + n)); + const path = utf8.decode(u8.subarray(p + 2, p + 2 + n)); p += 2 + n; const size = dv.getUint32(p); files.set(path, u8.subarray(p + 4, p + 4 + size)); @@ -38,7 +39,7 @@ export function parsePack(buf: ArrayBuffer): Pack { const raw = files.get("/index.prefix") ?? files.get("/index.txt"); if (raw) { pack.index = new PackIndex(pack); - addIndex(pack, dec.decode(raw), files.has("/index.prefix")); + addIndex(pack, utf8.decode(raw), files.has("/index.prefix")); files.delete("/index.txt"); files.delete("/index.prefix"); for (const [p, data] of files) { @@ -103,7 +104,7 @@ export class PackIndex extends Map { const raw = this.modules.get(module); if (!raw) return; this.modules.delete(module); - addIndex(this.pack, new TextDecoder().decode(raw), true); + addIndex(this.pack, utf8.decode(raw), true); } }