Speed up zip reading and writing: Inflate, CRC32, array intrinsics, receiver-class inlining

src/fs.ts

@@ -223,7 +223,10 @@ export class VFS {
   }
 
   private changed(path: string, event: "create" | "modify" | "delete") {
-    path = normalize(path);
+    this.changedNormalized(normalize(path), event);
+  }
+
+  private changedNormalized(path: string, event: "create" | "modify" | "delete") {
     for (const watcher of this.watchers) watcher(path, event);
     this.warmChanged?.add(path);
   }
@@ -320,13 +323,14 @@ export class VFS {
     if (!r || ((write || truncate) && !r[0].writable) || !r[0].openFile) return undefined;
     const h = r[0].openFile(r[1], write, truncate, readable);
     // writes through the handle are changes watchers see, as writes through VFS.write are
+    const changed = normalize(path);
     return {
       size: () => h.size(),
       ino: () => h.ino(),
       read: (dst, off, len, pos) => h.read(dst, off, len, pos),
       close: () => h.close(),
-      write: (src, pos) => { h.write(src, pos); this.changed(path, "modify"); },
-      truncate: (size) => { h.truncate(size); this.changed(path, "modify"); },
+      write: (src, pos) => { h.write(src, pos); this.changedNormalized(changed, "modify"); },
+      truncate: (size) => { h.truncate(size); this.changedNormalized(changed, "modify"); },
     };
   }
 

src/inflate.ts

@@ -25,15 +25,15 @@ function adler(d: Uint8Array): number {
 
 // Lookup table indexed by the next maxBits bits: entry = symbol << 4 | length.
 interface Table {
-  t: Int32Array;
+  t: Uint16Array;
   bits: number;
 }
 
 function buildTable(lengths: ArrayLike<number>, n: number): Table {
   let max = 0;
   for (let i = 0; i < n; i++) if (lengths[i] > max) max = lengths[i];
   const bits = Math.max(max, 1);
-  const t = new Int32Array(1 << bits);
+  const t = new Uint16Array(1 << bits);
   const count = new Array(16).fill(0);
   for (let i = 0; i < n; i++) count[lengths[i]]++;
   count[0] = 0;
@@ -90,6 +90,9 @@ export class Inflate {
   private out!: Uint8Array;
   private op = 0;
   private oe = 0;
+  private start = 0;
+  // output before this has been added to the checksum
+  private summed = 0;
 
   constructor(nowrap: boolean) {
     this.wrap = !nowrap;
@@ -150,6 +153,8 @@ export class Inflate {
     this.ip = 0;
     this.out = output;
     this.op = off;
+    this.start = off;
+    this.summed = off;
     this.oe = off + len;
     let ip = 0, buf = 0, cnt = 0;
     try {
@@ -168,9 +173,40 @@ export class Inflate {
       this.ip -= this.bitcnt >> 3;
       this.bitbuf = this.bitcnt = 0;
     }
+    this.finishCall();
     return [this.ip, this.op - off];
   }
 
+  private sum() {
+    const out = this.out, op = this.op;
+    if (!this.wrap) return;
+    let s1 = this.s1, s2 = this.s2;
+    for (let i = this.summed; i < op;) {
+      const end = Math.min(op, i + 3800);
+      for (; i < end; i++) { s1 += out[i]; s2 += s1; }
+      s1 %= 65521;
+      s2 %= 65521;
+    }
+    this.s1 = s1;
+    this.s2 = s2;
+    this.summed = op;
+  }
+
+  // The window and checksum take the call's output in one pass, not per byte.
+  private finishCall() {
+    const out = this.out, op = this.op, win = this.win;
+    this.sum();
+    let from = Math.max(this.start, op - 32768);
+    let abs = this.wpos + (from - this.start);
+    this.wpos += op - this.start;
+    while (from < op) {
+      const at = abs & 32767, n = Math.min(op - from, 32768 - at);
+      win.set(out.subarray(from, from + n), at);
+      from += n;
+      abs += n;
+    }
+  }
+
   private need(n: number) {
     while (this.bitcnt < n) {
       if (this.ip >= this.in.length) throw UNDERFLOW;
@@ -205,17 +241,6 @@ export class Inflate {
     return e >> 4;
   }
 
-  private adlerByte(b: number) {
-    this.s1 = (this.s1 + b) % 65521;
-    this.s2 = (this.s2 + this.s1) % 65521;
-  }
-
-  private emit(b: number) {
-    this.out[this.op++] = b;
-    this.win[this.wpos++ & 32767] = b;
-    if (this.wrap) this.adlerByte(b);
-  }
-
   private align() {
     const drop = this.bitcnt & 7;
     this.bitbuf >>>= drop;
@@ -262,16 +287,24 @@ export class Inflate {
       case Stage.Stored: {
         while (this.stored > 0) {
           if (this.op >= this.oe) return false;
-          if (this.bitcnt >= 8) this.emit(this.bits(8));
-          else if (this.ip < this.in.length) this.emit(this.in[this.ip++]);
-          else return false;
+          if (this.bitcnt >= 8) this.out[this.op++] = this.bits(8);
+          else {
+            const n = Math.min(this.stored, this.in.length - this.ip, this.oe - this.op);
+            if (n === 0) return false;
+            this.out.set(this.in.subarray(this.ip, this.ip + n), this.op);
+            this.ip += n;
+            this.op += n;
+            this.stored -= n;
+            continue;
+          }
           this.stored--;
         }
         this.stage = Stage.Block;
         return true;
       }
       case Stage.Trailer: {
         this.align();
+        this.sum();
         if (this.u32be() !== this.adler) throw new DataFormatError("incorrect data check");
         this.stage = Stage.Done;
         return false;
@@ -313,45 +346,87 @@ export class Inflate {
     this.dist = buildTable(lens.subarray(hlit), hdist);
   }
 
+  // The hot loop, on locals. A symbol that the input cannot finish is backed
+  // out, so the next call decodes it again from its first bit. The window is
+  // behind the call's own output, which a match reads directly.
   private huffman(): boolean {
-    const win = this.win;
-    while (this.copyLen > 0) {
-      if (this.op >= this.oe) return false;
-      this.emit(win[(this.wpos - this.copyDist) & 32767]);
-      this.copyLen--;
-    }
+    const input = this.in, out = this.out, win = this.win, oe = this.oe, start = this.start;
+    const lit = this.lit!, dist = this.dist!;
+    const lt = lit.t, lbits = lit.bits, lmask = (1 << lbits) - 1;
+    const dt = dist.t, dbits = dist.bits, dmask = (1 << dbits) - 1;
+    const end = input.length;
+    // absolute position of out[i] is base + i
+    const base = this.wpos - start;
+    let ip = this.ip, buf = this.bitbuf, cnt = this.bitcnt, op = this.op;
+    let copyLen = this.copyLen;
+    const copyDist = this.copyDist;
+    let dd = copyDist;
     for (;;) {
-      if (this.op >= this.oe) return false;
-      const ip = this.ip, buf = this.bitbuf, cnt = this.bitcnt;
-      try {
-        const sym = this.decode(this.lit!);
-        if (sym < 256) {
-          this.emit(sym);
-          continue;
-        }
-        if (sym === 256) {
-          this.stage = Stage.Block;
-          return true;
-        }
-        const li = sym - 257;
-        if (li >= 29) throw new DataFormatError("invalid literal/length code");
-        const len = LEN_BASE[li] + this.bits(LEN_EXTRA[li]);
-        const ds = this.decode(this.dist!);
-        if (ds >= 30) throw new DataFormatError("invalid distance code");
-        const dist = DIST_BASE[ds] + this.bits(DIST_EXTRA[ds]);
-        if (dist > this.wpos) throw new DataFormatError("invalid distance too far back");
-        this.copyLen = len;
-        this.copyDist = dist;
-      } catch (e) {
-        if (e !== UNDERFLOW) throw e;
-        this.ip = ip; this.bitbuf = buf; this.bitcnt = cnt;
-        return false;
+      if (copyLen > 0) {
+        let n = Math.min(copyLen, oe - op);
+        copyLen -= n;
+        let s = op - dd;
+        for (; n > 0 && s < start; n--) out[op++] = win[(base + s++) & 32767];
+        if (n > 32 && dd >= n) {
+          out.copyWithin(op, s, s + n);
+          op += n;
+        } else for (; n > 0; n--) out[op++] = out[s++];
+        if (copyLen > 0) break;
+      }
+      const sip = ip, sbuf = buf, scnt = cnt;
+      while (cnt <= 23 && ip < end) { buf |= input[ip++] << cnt; cnt += 8; }
+      let e = lt[buf & lmask];
+      let len = e & 15;
+      if (len === 0) {
+        if (cnt < lbits) { ip = sip; buf = sbuf; cnt = scnt; break; }
+        this.ip = ip; this.bitbuf = buf; this.bitcnt = cnt; this.op = op;
+        throw new DataFormatError("invalid code");
+      }
+      if (len > cnt) { ip = sip; buf = sbuf; cnt = scnt; break; }
+      buf >>= len;
+      cnt -= len;
+      const sym = e >> 4;
+      if (sym < 256) {
+        // a full output stops before the next literal but still sees the end of block, as zlib does
+        if (op >= oe) { ip = sip; buf = sbuf; cnt = scnt; break; }
+        out[op++] = sym;
+        continue;
+      }
+      if (sym === 256) {
+        this.stage = Stage.Block;
+        this.ip = ip; this.bitbuf = buf; this.bitcnt = cnt; this.op = op; this.copyLen = 0;
+        return true;
       }
-      while (this.copyLen > 0) {
-        if (this.op >= this.oe) return false;
-        this.emit(win[(this.wpos - this.copyDist) & 32767]);
-        this.copyLen--;
+      const li = sym - 257;
+      if (li >= 29) throw new DataFormatError("invalid literal/length code");
+      const eb = LEN_EXTRA[li];
+      if (cnt < eb) { ip = sip; buf = sbuf; cnt = scnt; break; }
+      const mlen = LEN_BASE[li] + (buf & ((1 << eb) - 1));
+      buf >>= eb;
+      cnt -= eb;
+      while (cnt <= 23 && ip < end) { buf |= input[ip++] << cnt; cnt += 8; }
+      e = dt[buf & dmask];
+      len = e & 15;
+      if (len === 0) {
+        if (cnt < dbits) { ip = sip; buf = sbuf; cnt = scnt; break; }
+        throw new DataFormatError("invalid code");
       }
+      if (len > cnt) { ip = sip; buf = sbuf; cnt = scnt; break; }
+      buf >>= len;
+      cnt -= len;
+      const ds = e >> 4;
+      if (ds >= 30) throw new DataFormatError("invalid distance code");
+      const deb = DIST_EXTRA[ds];
+      while (cnt <= 23 && ip < end) { buf |= input[ip++] << cnt; cnt += 8; }
+      if (cnt < deb) { ip = sip; buf = sbuf; cnt = scnt; break; }
+      dd = DIST_BASE[ds] + (buf & ((1 << deb) - 1));
+      buf >>= deb;
+      cnt -= deb;
+      if (dd > base + op) throw new DataFormatError("invalid distance too far back");
+      copyLen = mlen;
+      this.copyDist = dd;
     }
+    this.ip = ip; this.bitbuf = buf; this.bitcnt = cnt; this.op = op; this.copyLen = copyLen;
+    return false;
   }
 }

src/jit.ts

@@ -999,18 +999,27 @@ function fuseConcat(g: Gen, b: Body, pc: number): { pieces: string[]; end: numbe
   return { pieces: pieces.map((f) => f()), end: last + 3 };
 }
 
-// An invoke inside a region: the target must be fixed and inlinable itself.
+// An invoke inside a region: the target must be fixed and inlinable itself, or a
+// virtual call to the receiver class seen last here, which deoptimizes on any other.
 function regionInvoke(g: Gen, b: Body, op: number, pc: number, d: number, target: JMethod | undefined): string {
-  if (!target || target.polymorphic || op === 0xb9 || (op === 0xb8 && target.cls.state !== INITIALIZED)) throw new NoInline();
-  const callee = op === 0xb7 ? g.t.vm.specialTarget(b.m.cls, target) : target;
-  if (!callee.code || callee.isAbstract || !isFixed(op, callee)) throw new NoInline();
+  if (!target || target.polymorphic || (op === 0xb8 && target.cls.state !== INITIALIZED)) throw new NoInline();
+  let callee = op === 0xb7 ? g.t.vm.specialTarget(b.m.cls, target) : target;
+  let guard: JClass | null = null;
+  if (op === 0xb9 || !isFixed(op, callee)) {
+    const site = b.m.cache?.[pc];
+    if (!site || !site.m.code || site.m.isAbstract || site.m.polymorphic) throw new NoInline();
+    callee = site.m;
+    guard = site.c;
+  }
+  if (!callee.code || callee.isAbstract) throw new NoInline();
   const n = target.argSlots;
   const base = d - n;
   const args = Array.from({ length: n }, (_, i) => b.S(base + i));
   const result = target.ret === "V" ? null : b.S(base);
+  const check = guard === null ? "" : `if(${args[0]}!==null&&${args[0]}.cls!==${g.K(guard)}){dpc=${pc};dsp=${d};throw rt.DEOPT;}`;
   const trivial = trivialBody(callee);
-  if (trivial) return genInline(trivial, result ?? "v", args, pc, callee.isStatic);
-  return (callee.isStatic ? "" : `if(${args[0]}===null)rt.npe(${pc});`) + genRegion(g, b, callee, args, result, pc, base);
+  if (trivial) return check + genInline(trivial, result ?? "v", args, pc, callee.isStatic);
+  return check + (callee.isStatic ? "" : `if(${args[0]}===null)rt.npe(${pc});`) + genRegion(g, b, callee, args, result, pc, base);
 }
 
 // The callee of an invoke does not depend on the receiver's class.
@@ -1150,19 +1159,32 @@ function genInline(body: Trivial, res: string, args: string[], pc: number, isSta
 
 function genInvoke(
   g: Gen, op: number, pc: number, next: number, d: number, target: JMethod, cls: JClass,
-  slowDepth = d, inline = op !== 0xb9 && slowDepth === d,
+  slowDepth = d, inline = slowDepth === d,
 ): string {
   const { t, K, exit, call } = g;
   const n = target.argSlots;
   const base = d - n;
   const args = Array.from({ length: n }, (_, i) => `s${base + i}`);
   // inline the callee, guarded by the receiver's class unless the target is fixed
   if (inline) {
-    const callee = op === 0xb7 ? t.vm.specialTarget(cls, target) : target;
+    let callee = op === 0xb7 ? t.vm.specialTarget(cls, target) : target;
+    // the class guarding the inlined body, or null when the callee is fixed
+    let guard: JClass | null = isFixed(op, callee) && op !== 0xb9 ? null : callee.cls;
+    // a virtual or interface call inlines the method of the receiver class seen last at this site
+    let seen = false;
+    if (guard !== null && !target.isPrivate) {
+      const site = g.top.m.cache?.[pc];
+      if (site && site.m.code && !site.m.isAbstract && !site.m.polymorphic) {
+        callee = site.m;
+        guard = site.c;
+        seen = true;
+      }
+    }
     const res = `s${base}`;
     let code: string | null = null;
     const trivial = trivialBody(callee);
-    if (trivial) code = genInline(trivial, res, args, pc, callee.isStatic);
+    if (op === 0xb9 && !seen) code = null;
+    else if (trivial) code = genInline(trivial, res, args, pc, callee.isStatic);
     else if (callee.code && !callee.isAbstract) {
       try {
         const nullCheck = callee.isStatic ? "" : `if(${args[0]}===null)rt.npe(${pc});`;
@@ -1172,8 +1194,8 @@ function genInvoke(
       }
     }
     if (code !== null) {
-      if (isFixed(op, callee)) return code;
-      return `if(${args[0]}!==null&&${args[0]}.cls===${K(callee.cls)}){${code}}else{${genInvoke(g, op, pc, next, d, target, cls, d, false)}}`;
+      if (guard === null) return code;
+      return `if(${args[0]}!==null&&${args[0]}.cls===${K(guard)}){${code}}else{${genInvoke(g, op, pc, next, d, target, cls, d, false)}}`;
     }
   }
   const parts: string[] = [`f.pc=${pc};`];