diff --git a/docs/support.md b/docs/support.md index 504cb00..54ed54b 100644 --- a/docs/support.md +++ b/docs/support.md @@ -14,9 +14,10 @@ java-web targets headless Java: console apps, libraries and javac. shared with direct buffers and other native allocations. A mapping that does not fit fails with `IOException: Map failed` and an `OutOfMemoryError` cause. Read-write changes reach the file on force or unmap. -- `Deflater` writes valid DEFLATE streams using stored blocks. Compression - levels and strategies currently do not reduce output size; compressed bytes - and compression ratios differ from HotSpot's zlib output. +- `Deflater` is a DEFLATE compressor in TypeScript (`src/deflate.ts`) with + zlib's levels, strategies, flush modes, dictionaries and `setLevel` and + `setStrategy` changes. Compressed sizes are close to zlib's at each level, but + the bytes are not the same as HotSpot's zlib output. - Child processes (`ProcessBuilder`, `Runtime.exec`, `ProcessHandle`) work under Node and run the host's programs. Starting `java` starts another java-web VM on the same image. A child cannot use a directory that exists only diff --git a/src/deflate.ts b/src/deflate.ts new file mode 100644 index 0000000..972a6e3 --- /dev/null +++ b/src/deflate.ts @@ -0,0 +1,768 @@ +// Streaming DEFLATE encoder (RFC 1950/1951) for java.util.zip.Deflater. +// Hash chains over a 32 KB window find matches; levels 1-3 take the first +// match, 4-9 look one byte ahead. Blocks use dynamic or fixed Huffman codes, +// or are stored when that is smaller. Compressed bytes sit in a pending buffer +// that the caller drains, so memory stays bounded. + +function adler32(adler: number, b: Uint8Array): number { + let s1 = adler & 0xffff, s2 = adler >>> 16; + for (let i = 0; i < b.length;) { + const end = Math.min(b.length, i + 3800); + for (; i < end; i++) { s1 += b[i]; s2 += s1; } + s1 %= 65521; + s2 %= 65521; + } + return ((s2 << 16) | s1) | 0; +} + +const LEN_BASE = [3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258]; +const LEN_EXTRA = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0]; +const DIST_BASE = [1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, + 4097, 6145, 8193, 12289, 16385, 24577]; +const DIST_EXTRA = [0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13]; +const CL_ORDER = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]; + +const WSIZE = 32768; +const WMASK = WSIZE - 1; +const MIN_MATCH = 3; +const MAX_MATCH = 258; +const MIN_LOOKAHEAD = MAX_MATCH + MIN_MATCH + 1; +const MAX_DIST = WSIZE - MIN_LOOKAHEAD; +const TOO_FAR = 4096; +const SYMS = 16383; +const STORED_MAX = 65535; +// largest block (fixed codes, 31 bits a symbol) or stored block, plus a flush marker and trailer +const PENDING = 65600; + +const NO_FLUSH = 0, FINISH = 4, BLOCK = 5; +const SYNC_FLUSH = 2, FULL_FLUSH = 3; +const FILTERED = 1, HUFFMAN_ONLY = 2; + +const NEED = 0, BLOCKED = 1, DONE = 2; + +// good, lazy (longest match to insert, in levels 1-3), nice, chain +const CONFIG = [ + [0, 0, 0, 0], [4, 4, 8, 4], [4, 5, 16, 8], [4, 6, 32, 32], [4, 4, 16, 16], + [8, 16, 32, 32], [8, 16, 128, 128], [8, 32, 128, 256], [32, 128, 258, 1024], [32, 258, 258, 4096], +]; + +const LCODE = new Uint8Array(256); +const DCODE = new Uint8Array(512); +const LBASE0 = LEN_BASE.map((b) => b - 3); +for (let c = 0; c < 28; c++) for (let k = 0; k < 1 << LEN_EXTRA[c]; k++) LCODE[LBASE0[c] + k] = c; +LCODE[255] = 28; +for (let c = 0; c < 30; c++) { + for (let k = 0; k < 1 << DIST_EXTRA[c]; k++) { + const d = DIST_BASE[c] + k - 1; + DCODE[d < 256 ? d : 256 + (d >> 7)] = c; + } +} + +const CNT = new Uint16Array(16); +const NEXT = new Uint16Array(16); + +// canonical codes, bit-reversed for the LSB-first bit order +function makeCodes(lens: Uint8Array, n: number, codes: Uint16Array) { + CNT.fill(0); + for (let i = 0; i < n; i++) CNT[lens[i]]++; + CNT[0] = 0; + let code = 0; + for (let len = 1; len <= 15; len++) { + code = (code + CNT[len - 1]) << 1; + NEXT[len] = code; + } + for (let i = 0; i < n; i++) { + const l = lens[i]; + if (!l) continue; + let c = NEXT[l]++, r = 0; + for (let k = 0; k < l; k++) { r = (r << 1) | (c & 1); c >>= 1; } + codes[i] = r; + } +} + +const FIXED_LIT_LEN = new Uint8Array(288); +FIXED_LIT_LEN.fill(8, 0, 144); +FIXED_LIT_LEN.fill(9, 144, 256); +FIXED_LIT_LEN.fill(7, 256, 280); +FIXED_LIT_LEN.fill(8, 280, 288); +const FIXED_DIST_LEN = new Uint8Array(30).fill(5); +const FIXED_LIT_CODE = new Uint16Array(288); +const FIXED_DIST_CODE = new Uint16Array(30); +makeCodes(FIXED_LIT_LEN, 288, FIXED_LIT_CODE); +makeCodes(FIXED_DIST_LEN, 30, FIXED_DIST_CODE); + +// scratch for the code builders +const KEYS = new Uint32Array(288); +const IW = new Uint32Array(288); +const LPAR = new Uint16Array(288); +const IPAR = new Uint16Array(288); +const DEP = new Uint16Array(288); +const BLC = new Uint16Array(320); +const LIT_LEN = new Uint8Array(288); +const LIT_CODE = new Uint16Array(288); +const DIST_LEN = new Uint8Array(32); +const DIST_CODE = new Uint16Array(32); +const CL_LEN = new Uint8Array(19); +const CL_CODE = new Uint16Array(19); +const CL_FREQ = new Uint32Array(19); +const SEQ = new Uint8Array(320); +const RLE_SYM = new Uint8Array(320); +const RLE_X = new Uint8Array(320); + +// Code lengths of at most maxBits for the symbols with a frequency. Every +// tree gets at least two codes, as zlib does, so decoders see a complete code. +function buildLengths(freq: Uint32Array, n: number, maxBits: number, lens: Uint8Array) { + lens.fill(0, 0, n); + let m = 0; + for (let s = 0; s < n; s++) if (freq[s]) KEYS[m++] = (freq[s] << 9) | s; + for (let s = 0; m < 2 && s < n; s++) if (!freq[s]) KEYS[m++] = (1 << 9) | s; + KEYS.subarray(0, m).sort(); + // two-queue Huffman: leaves ascending, merged nodes created in ascending order + let leaf = 0, q = 0; + for (let made = 0; made < m - 1; made++) { + let sum = 0; + for (let k = 0; k < 2; k++) { + if (leaf < m && (q >= made || KEYS[leaf] >>> 9 <= IW[q])) { + sum += KEYS[leaf] >>> 9; + LPAR[leaf++] = made; + } else { + sum += IW[q]; + IPAR[q++] = made; + } + } + IW[made] = sum; + } + DEP[m - 2] = 0; + for (let i = m - 3; i >= 0; i--) DEP[i] = DEP[IPAR[i]] + 1; + BLC.fill(0); + let maxDepth = 0; + for (let i = 0; i < m; i++) { + const d = DEP[LPAR[i]] + 1; + BLC[d]++; + if (d > maxDepth) maxDepth = d; + } + if (maxDepth > maxBits) { + for (let d = maxBits + 1; d <= maxDepth; d++) { + BLC[maxBits] += BLC[d]; + BLC[d] = 0; + } + let kraft = 0; + for (let d = 1; d <= maxBits; d++) kraft += BLC[d] << (maxBits - d); + // each step trades one longest code for a split of a shorter one + for (; kraft > 1 << maxBits; kraft--) { + BLC[maxBits]--; + for (let i = maxBits - 1; i > 0; i--) { + if (BLC[i]) { + BLC[i]--; + BLC[i + 1] += 2; + break; + } + } + } + } + let i = 0; + for (let d = maxBits; d >= 1; d--) for (let c = BLC[d]; c > 0; c--) lens[KEYS[i++] & 511] = d; +} + +function dataBits(ll: Uint8Array, dl: Uint8Array, lit: Uint32Array, dist: Uint32Array): number { + let bits = 0; + for (let s = 0; s < 286; s++) bits += lit[s] * ll[s]; + for (let c = 0; c < 29; c++) bits += lit[257 + c] * LEN_EXTRA[c]; + for (let d = 0; d < 30; d++) bits += dist[d] * (dl[d] + DIST_EXTRA[d]); + return bits; +} + +const EMPTY8 = new Uint8Array(0); +const EMPTY16 = new Uint16Array(0); + +export class Deflate { + private readonly wrap: boolean; + private level: number; + private strategy: number; + private headerDone = false; + private ended = false; + private lastFlush = -2; + private adler = 1; + private dictId: number | null = null; + // output waiting for room, and bits not yet a whole byte + private pending = EMPTY8; + private pendPos = 0; + private pendLen = 0; + private bitBuf = 0; + private bitCnt = 0; + // window: data at [0, strstart + lookahead); at level 0 it holds the stored block being filled + private win = EMPTY8; + private head = EMPTY16; + private prev = EMPTY16; + private strstart = 0; + private lookahead = 0; + private blockStart = 0; + private hashed = 0; + private matchStart = 0; + private matchLen = MIN_MATCH - 1; + private prevLen = MIN_MATCH - 1; + private prevMatch = 0; + private matchAvail = false; + private heldLen = 0; + // symbols of the block being built: distance (0 for a literal) and literal or length - 3 + private symD = EMPTY16; + private symL = EMPTY8; + private nsym = 0; + private litFreq = new Uint32Array(0); + private distFreq = new Uint32Array(0); + // input of the current call + private inp: Uint8Array = EMPTY8; + private inPos = 0; + + constructor(level: number, strategy: number, nowrap: boolean) { + this.wrap = !nowrap; + this.level = level < 0 ? 6 : level; + this.strategy = strategy; + } + + get finished() { return this.ended && this.pendPos === this.pendLen; } + get checksum() { return this.headerDone || this.dictId === null ? this.adler : this.dictId; } + + reset() { + this.headerDone = this.ended = false; + this.pendPos = this.pendLen = this.bitBuf = this.bitCnt = this.heldLen = 0; + this.lastFlush = -2; + this.adler = 1; + this.dictId = null; + if (this.win.length) this.resetHistory(); + } + + private alloc() { + if (this.win.length) return; + this.win = new Uint8Array(2 * WSIZE + MAX_MATCH + 8); + this.head = new Uint16Array(1 << 15); + this.prev = new Uint16Array(WSIZE); + this.symD = new Uint16Array(SYMS + 1); + this.symL = new Uint8Array(SYMS + 1); + this.litFreq = new Uint32Array(286); + this.distFreq = new Uint32Array(30); + this.pending = new Uint8Array(PENDING); + } + + private resetHistory() { + this.strstart = this.lookahead = this.blockStart = this.hashed = this.nsym = 0; + this.matchLen = this.prevLen = MIN_MATCH - 1; + this.matchAvail = false; + this.head.fill(0); + this.litFreq.fill(0); + this.distFreq.fill(0); + } + + // false when a zlib stream can no longer take a dictionary + setDictionary(d: Uint8Array): boolean { + if (this.wrap && this.headerDone && !this.ended) return false; + if (this.lookahead) return false; + if (this.wrap && !this.headerDone) this.dictId = adler32(1, d) >>> 0; + if (this.level === 0 || this.ended) return true; + this.alloc(); + this.resetHistory(); + const t = d.length > WSIZE ? d.subarray(d.length - WSIZE) : d; + this.win.set(t, 0); + this.strstart = this.blockStart = t.length; + let p = 0; + for (; p + 2 < t.length; p++) this.insertAt(p); + this.hashed = p; + return true; + } + + private put(...bytes: number[]) { + for (const b of bytes) this.pending[this.pendLen++] = b; + } + + private be32(v: number) { + this.put(v >>> 24, (v >>> 16) & 0xff, (v >>> 8) & 0xff, v & 0xff); + } + + private putBits(v: number, n: number) { + this.bitBuf |= v << this.bitCnt; + this.bitCnt += n; + while (this.bitCnt >= 8) { + this.pending[this.pendLen++] = this.bitBuf & 0xff; + this.bitBuf >>>= 8; + this.bitCnt -= 8; + } + } + + private align() { + if (this.bitCnt > 0) { + this.pending[this.pendLen++] = this.bitBuf & 0xff; + this.bitBuf = this.bitCnt = 0; + } + } + + private drain(output: Uint8Array, at: number): number { + const n = Math.min(output.length - at, this.pendLen - this.pendPos); + output.set(this.pending.subarray(this.pendPos, this.pendPos + n), at); + this.pendPos += n; + if (this.pendPos === this.pendLen) this.pendPos = this.pendLen = 0; + return n; + } + + private header() { + this.headerDone = true; + if (!this.wrap) return; + const flags = this.strategy >= HUFFMAN_ONLY || this.level < 2 ? 0 : this.level < 6 ? 1 : this.level === 6 ? 2 : 3; + let h = 0x7800 | (flags << 6); + if (this.dictId !== null) h |= 0x20; + h += 31 - (h % 31); + this.put(h >> 8, h & 0xff); + if (this.dictId !== null) this.be32(this.dictId); + } + + // Compresses what fits. flush: 0 none, 2 sync, 3 full, 4 finish. + // Returns [read, written, retry]; retry means a parameter change is not done. + run(input: Uint8Array, output: Uint8Array, flush: number, params: number): [number, number, boolean] { + if (params & 1) return this.setParams(input, output, params >> 3, (params >> 1) & 3); + const [read, written] = this.deflate(input, output, flush); + return [read, written, false]; + } + + private setParams(input: Uint8Array, output: Uint8Array, level: number, strategy: number): [number, number, boolean] { + if (level < 0) level = 6; + const family = (l: number) => (l === 0 ? 0 : l < 4 ? 1 : 2); + let read = 0, written = 0; + // a different matcher ends the current block first, as zlib's deflateParams does + if (this.lastFlush !== -2 && (strategy !== this.strategy || family(level) !== family(this.level))) { + [read, written] = this.deflate(input, output, BLOCK); + if (read < input.length || this.pendLen || !this.idle()) return [read, written, true]; + } + const stored = (this.level === 0) !== (level === 0); + this.level = level; + this.strategy = strategy; + if (stored && this.win.length) this.resetHistory(); + return [read, written, false]; + } + + private idle() { + return this.lookahead === 0 && this.nsym === 0 && !this.matchAvail && this.strstart === this.blockStart && this.heldLen === 0; + } + + private deflate(input: Uint8Array, output: Uint8Array, flush: number): [number, number] { + if (output.length === 0) return [0, 0]; + let written = this.drain(output, 0); + if (this.pendLen) { + this.lastFlush = -1; + return [0, written]; + } + // a repeated flush with no new input adds nothing, so flush loops end + if (input.length === 0 && flush !== BLOCK && flush !== FINISH && flush <= this.lastFlush) return [0, written]; + if (flush !== BLOCK) this.lastFlush = flush; + this.alloc(); + this.inp = input; + this.inPos = 0; + let marked = false; + for (;;) { + written += this.drain(output, written); + if (this.pendLen || this.ended) break; + if (!this.headerDone) { + this.header(); + continue; + } + const st = this.level === 0 ? this.storedStep(flush) : this.strategy === HUFFMAN_ONLY ? this.huff(flush) + : this.level < 4 ? this.fast(flush) : this.lazy(flush); + if (st === BLOCKED) continue; + if (st === NEED || marked) break; + marked = true; + if (flush === FINISH) { + if (this.wrap) this.be32(this.adler >>> 0); + this.ended = true; + } else if (flush === SYNC_FLUSH || flush === FULL_FLUSH) { + this.putBits(0, 3); + this.align(); + this.put(0, 0, 0xff, 0xff); + if (flush === FULL_FLUSH && this.level !== 0) this.forget(); + } else break; + } + const read = this.inPos; + this.inp = EMPTY8; + return [read, written]; + } + + // a full flush drops the match history + private forget() { + this.head.fill(0); + this.matchLen = this.prevLen = MIN_MATCH - 1; + this.matchAvail = false; + if (this.lookahead === 0) this.strstart = this.blockStart = this.hashed = 0; + else this.hashed = this.strstart; + } + + private storedStep(flush: number): number { + while (this.inPos < this.inp.length) { + const n = Math.min(STORED_MAX - this.heldLen, this.inp.length - this.inPos); + const chunk = this.inp.subarray(this.inPos, this.inPos + n); + this.win.set(chunk, this.heldLen); + if (this.wrap) this.adler = adler32(this.adler, chunk); + this.heldLen += n; + this.inPos += n; + if (this.heldLen === STORED_MAX) { + this.writeStored(0, this.heldLen, false); + this.heldLen = 0; + return BLOCKED; + } + } + if (flush === NO_FLUSH) return NEED; + if (flush === FINISH || this.heldLen) { + this.writeStored(0, this.heldLen, flush === FINISH); + this.heldLen = 0; + } + return DONE; + } + + private fill() { + const win = this.win; + while (this.lookahead < MIN_LOOKAHEAD && this.inPos < this.inp.length) { + if (this.strstart >= WSIZE + MAX_DIST) this.slide(); + const at = this.strstart + this.lookahead; + const n = Math.min(2 * WSIZE - at, this.inp.length - this.inPos); + const chunk = this.inp.subarray(this.inPos, this.inPos + n); + win.set(chunk, at); + if (this.wrap) this.adler = adler32(this.adler, chunk); + this.lookahead += n; + this.inPos += n; + } + } + + private slide() { + this.win.copyWithin(0, WSIZE, this.strstart + this.lookahead); + this.matchStart -= WSIZE; + this.strstart -= WSIZE; + this.blockStart -= WSIZE; + this.hashed = Math.max(0, this.hashed - WSIZE); + const head = this.head, prev = this.prev; + for (let i = 0; i < head.length; i++) head[i] = head[i] >= WSIZE ? head[i] - WSIZE : 0; + for (let i = 0; i < prev.length; i++) prev[i] = prev[i] >= WSIZE ? prev[i] - WSIZE : 0; + } + + // returns the previous head of the chain for the 3 bytes at p + private insertAt(p: number): number { + const w = this.win; + const h = Math.imul(w[p] | (w[p + 1] << 8) | (w[p + 2] << 16), 0x9e3779b1) >>> 17; + const old = this.head[h]; + this.prev[p & WMASK] = old; + this.head[h] = p; + return old; + } + + private insertUpTo(end: number) { + const dataEnd = this.strstart + this.lookahead; + while (this.hashed < end && this.hashed + 2 < dataEnd) this.insertAt(this.hashed++); + } + + // Longest match for the bytes at strstart among the chain from cand, longer than best. + private longest(cand: number, best: number, chain: number, nice: number): number { + const win = this.win, prev = this.prev, cur = this.strstart; + const limit = cur > MAX_DIST ? cur - MAX_DIST : 0; + const maxLen = this.lookahead < MAX_MATCH ? this.lookahead : MAX_MATCH; + if (nice > maxLen) nice = maxLen; + const c0 = win[cur], c1 = win[cur + 1]; + do { + if (win[cand + best] === win[cur + best] && win[cand] === c0 && win[cand + 1] === c1 + && win[cand + best - 1] === win[cur + best - 1]) { + let l = 2; + while (l < maxLen && win[cand + l] === win[cur + l]) l++; + if (l > best) { + this.matchStart = cand; + best = l; + if (l >= nice) break; + } + } + cand = prev[cand & WMASK]; + } while (cand > limit && --chain !== 0); + return best > this.lookahead ? this.lookahead : best; + } + + private tallyLit(c: number): boolean { + const n = this.nsym++; + this.symD[n] = 0; + this.symL[n] = c; + this.litFreq[c]++; + return this.nsym === SYMS; + } + + private tallyMatch(dist: number, len: number): boolean { + const n = this.nsym++; + this.symD[n] = dist; + this.symL[n] = len - 3; + this.litFreq[257 + LCODE[len - 3]]++; + this.distFreq[dist <= 256 ? DCODE[dist - 1] : DCODE[256 + ((dist - 1) >> 7)]]++; + return this.nsym === SYMS; + } + + // the data is used up: close the block the flush mode asks for + private tail(flush: number): number { + if (this.matchAvail) { + this.tallyLit(this.win[this.strstart - 1]); + this.matchAvail = false; + } + if (flush === FINISH) this.flushBlock(true); + else if (this.nsym) this.flushBlock(false); + return DONE; + } + + private huff(flush: number): number { + const win = this.win; + for (;;) { + if (this.lookahead === 0) { + this.fill(); + if (this.lookahead === 0) return flush === NO_FLUSH ? NEED : this.tail(flush); + } + const full = this.tallyLit(win[this.strstart]); + this.lookahead--; + this.hashed = ++this.strstart; + if (full) { + this.flushBlock(false); + return BLOCKED; + } + } + } + + private fast(flush: number): number { + const win = this.win; + const cfg = CONFIG[this.level], maxInsert = cfg[1], nice = cfg[2], chain = cfg[3]; + for (;;) { + if (this.lookahead < MIN_LOOKAHEAD) { + this.fill(); + if (this.lookahead < MIN_LOOKAHEAD && flush === NO_FLUSH) return NEED; + if (this.lookahead === 0) return this.tail(flush); + } + if (this.hashed < this.strstart) this.insertUpTo(this.strstart); + let hh = 0; + if (this.lookahead >= MIN_MATCH) { + hh = this.insertAt(this.strstart); + this.hashed = this.strstart + 1; + } + let len = 0; + if (hh !== 0 && this.strstart - hh <= MAX_DIST) len = this.longest(hh, MIN_MATCH - 1, chain, nice); + let full: boolean; + if (len >= MIN_MATCH) { + full = this.tallyMatch(this.strstart - this.matchStart, len); + this.lookahead -= len; + const end = this.strstart + len; + if (len <= maxInsert && this.lookahead >= MIN_MATCH) { + for (let p = this.strstart + 1; p < end; p++) this.insertAt(p); + } + this.hashed = this.strstart = end; + } else { + full = this.tallyLit(win[this.strstart]); + this.lookahead--; + this.strstart++; + } + if (full) { + this.flushBlock(false); + return BLOCKED; + } + } + } + + private lazy(flush: number): number { + const win = this.win; + const cfg = CONFIG[this.level], good = cfg[0], maxLazy = cfg[1], nice = cfg[2], chainMax = cfg[3]; + const filtered = this.strategy === FILTERED; + for (;;) { + if (this.lookahead < MIN_LOOKAHEAD) { + this.fill(); + if (this.lookahead < MIN_LOOKAHEAD && flush === NO_FLUSH) return NEED; + if (this.lookahead === 0) return this.tail(flush); + } + if (this.hashed < this.strstart) this.insertUpTo(this.strstart); + let hh = 0; + if (this.lookahead >= MIN_MATCH) { + hh = this.insertAt(this.strstart); + this.hashed = this.strstart + 1; + } + this.prevLen = this.matchLen; + this.prevMatch = this.matchStart; + this.matchLen = MIN_MATCH - 1; + if (hh !== 0 && this.prevLen < maxLazy && this.strstart - hh <= MAX_DIST) { + let len = this.longest(hh, this.prevLen, this.prevLen >= good ? chainMax >> 2 : chainMax, nice); + if (len <= 5 && (filtered || (len === MIN_MATCH && this.strstart - this.matchStart > TOO_FAR))) len = MIN_MATCH - 1; + this.matchLen = len; + } + if (this.prevLen >= MIN_MATCH && this.matchLen <= this.prevLen) { + // the match found one byte back is the better one + const dataEnd = this.strstart + this.lookahead; + const full = this.tallyMatch(this.strstart - 1 - this.prevMatch, this.prevLen); + this.lookahead -= this.prevLen - 1; + const end = this.strstart - 1 + this.prevLen; + let p = this.strstart + 1; + for (; p < end && p + 2 < dataEnd; p++) this.insertAt(p); + this.hashed = p; + this.matchAvail = false; + this.matchLen = MIN_MATCH - 1; + this.strstart = end; + if (full) { + this.flushBlock(false); + return BLOCKED; + } + } else if (this.matchAvail) { + const full = this.tallyLit(win[this.strstart - 1]); + if (full) this.flushBlock(false); + this.strstart++; + this.lookahead--; + if (full) return BLOCKED; + } else { + this.matchAvail = true; + this.strstart++; + this.lookahead--; + } + } + } + + private writeStored(start: number, len: number, last: boolean) { + do { + const n = Math.min(len, STORED_MAX); + len -= n; + this.putBits(last && len === 0 ? 1 : 0, 3); + this.align(); + this.put(n & 0xff, n >> 8, ~n & 0xff, (~n >> 8) & 0xff); + this.pending.set(this.win.subarray(start, start + n), this.pendLen); + this.pendLen += n; + start += n; + } while (len > 0); + } + + private emit(litCode: Uint16Array, litLen: Uint8Array, distCode: Uint16Array, distLen: Uint8Array) { + const symD = this.symD, symL = this.symL, pend = this.pending, n = this.nsym; + let bb = this.bitBuf, bc = this.bitCnt, pl = this.pendLen; + for (let i = 0; i < n; i++) { + const d = symD[i], l = symL[i]; + if (d === 0) { + bb |= litCode[l] << bc; + bc += litLen[l]; + } else { + const lc = LCODE[l], sym = 257 + lc; + bb |= (litCode[sym] | ((l - LBASE0[lc]) << litLen[sym])) << bc; + bc += litLen[sym] + LEN_EXTRA[lc]; + } + while (bc >= 8) { + pend[pl++] = bb & 0xff; + bb >>>= 8; + bc -= 8; + } + if (d !== 0) { + const dc = d <= 256 ? DCODE[d - 1] : DCODE[256 + ((d - 1) >> 7)]; + bb |= distCode[dc] << bc; + bc += distLen[dc]; + while (bc >= 8) { + pend[pl++] = bb & 0xff; + bb >>>= 8; + bc -= 8; + } + const eb = DIST_EXTRA[dc]; + if (eb) { + bb |= (d - DIST_BASE[dc]) << bc; + bc += eb; + while (bc >= 8) { + pend[pl++] = bb & 0xff; + bb >>>= 8; + bc -= 8; + } + } + } + } + bb |= litCode[256] << bc; + bc += litLen[256]; + while (bc >= 8) { + pend[pl++] = bb & 0xff; + bb >>>= 8; + bc -= 8; + } + this.bitBuf = bb; + this.bitCnt = bc; + this.pendLen = pl; + } + + private flushBlock(last: boolean) { + const lit = this.litFreq, dist = this.distFreq; + lit[256] = 1; + buildLengths(lit, 286, 15, LIT_LEN); + buildLengths(dist, 30, 15, DIST_LEN); + let nlit = 286, ndist = 30; + while (nlit > 257 && LIT_LEN[nlit - 1] === 0) nlit--; + while (ndist > 1 && DIST_LEN[ndist - 1] === 0) ndist--; + SEQ.set(LIT_LEN.subarray(0, nlit), 0); + SEQ.set(DIST_LEN.subarray(0, ndist), nlit); + const nrle = this.runLengths(nlit + ndist); + buildLengths(CL_FREQ, 19, 7, CL_LEN); + let hclen = 19; + while (hclen > 4 && CL_LEN[CL_ORDER[hclen - 1]] === 0) hclen--; + let dynBits = 14 + 3 * hclen + dataBits(LIT_LEN, DIST_LEN, lit, dist); + for (let s = 0; s < 19; s++) dynBits += CL_FREQ[s] * (CL_LEN[s] + (s === 16 ? 2 : s === 17 ? 3 : s === 18 ? 7 : 0)); + const fixBits = dataBits(FIXED_LIT_LEN, FIXED_DIST_LEN, lit, dist); + const dynBytes = (dynBits + 10) >> 3, fixBytes = (fixBits + 10) >> 3; + const storedLen = this.strstart - this.blockStart; + if (this.blockStart >= 0 && storedLen + 4 <= Math.min(dynBytes, fixBytes)) { + this.writeStored(this.blockStart, storedLen, last); + } else if (fixBytes <= dynBytes) { + this.putBits((last ? 1 : 0) | 2, 3); + this.emit(FIXED_LIT_CODE, FIXED_LIT_LEN, FIXED_DIST_CODE, FIXED_DIST_LEN); + } else { + makeCodes(LIT_LEN, 286, LIT_CODE); + makeCodes(DIST_LEN, 30, DIST_CODE); + makeCodes(CL_LEN, 19, CL_CODE); + this.putBits((last ? 1 : 0) | 4, 3); + this.putBits(nlit - 257, 5); + this.putBits(ndist - 1, 5); + this.putBits(hclen - 4, 4); + for (let i = 0; i < hclen; i++) this.putBits(CL_LEN[CL_ORDER[i]], 3); + for (let i = 0; i < nrle; i++) { + const s = RLE_SYM[i]; + this.putBits(CL_CODE[s], CL_LEN[s]); + if (s === 16) this.putBits(RLE_X[i], 2); + else if (s === 17) this.putBits(RLE_X[i], 3); + else if (s === 18) this.putBits(RLE_X[i], 7); + } + this.emit(LIT_CODE, LIT_LEN, DIST_CODE, DIST_LEN); + } + if (last) this.align(); + lit.fill(0); + dist.fill(0); + this.nsym = 0; + this.blockStart = this.strstart; + } + + // run-length codes (16, 17, 18) for the n code lengths in SEQ; fills CL_FREQ + private runLengths(n: number): number { + CL_FREQ.fill(0); + let k = 0; + const out = (s: number, x: number) => { + RLE_SYM[k] = s; + RLE_X[k++] = x; + CL_FREQ[s]++; + }; + for (let i = 0; i < n;) { + const v = SEQ[i]; + let run = 1; + while (i + run < n && SEQ[i + run] === v) run++; + i += run; + if (v === 0) { + while (run >= 11) { + const t = Math.min(run, 138); + out(18, t - 11); + run -= t; + } + if (run >= 3) { + out(17, run - 3); + run = 0; + } + while (run-- > 0) out(0, 0); + } else { + out(v, 0); + run--; + while (run >= 3) { + const t = Math.min(run, 6); + out(16, t - 3); + run -= t; + } + while (run-- > 0) out(v, 0); + } + } + return k; + } +} diff --git a/src/natives/zip.ts b/src/natives/zip.ts index fc5ca29..c0e88af 100644 --- a/src/natives/zip.ts +++ b/src/natives/zip.ts @@ -1,5 +1,6 @@ // java.util.zip natives: Inflater, Deflater, CRC32, Adler32. +import { Deflate } from "../deflate.ts"; import { DataFormatError, Inflate } from "../inflate.ts"; import type { JArray } from "../runtime.ts"; import { JThread, PendingThrow } from "../thread.ts"; @@ -44,109 +45,6 @@ export function adler32(adler: number, b: Uint8Array): number { return ((s2 << 16) | s1) | 0; } -const BLOCK = 65535; - -// Deflate with stored blocks only: valid output, no compression yet. As in -// zlib, input is taken only while output drains, so memory stays bounded. -class Deflate { - private readonly wrap: boolean; - private headerDone = false; - // input not yet in a block, and output waiting for room - private held = new Uint8Array(BLOCK); - private heldLen = 0; - private pending = new Uint8Array(BLOCK + 32); - private pendPos = 0; - private pendLen = 0; - private adler = 1; - private dictId: number | null = null; - private lastFlush = 0; - private ended = false; - constructor(nowrap: boolean) { - this.wrap = !nowrap; - } - get finished() { return this.ended && this.pendPos === this.pendLen; } - get checksum() { return this.headerDone || this.dictId === null ? this.adler : this.dictId; } - reset() { - this.headerDone = this.ended = false; - this.heldLen = this.pendPos = this.pendLen = this.lastFlush = 0; - this.adler = 1; - this.dictId = null; - } - // the header names the dictionary; compression would also use it - setDictionary(d: Uint8Array) { - if (this.wrap && this.headerDone && !this.ended) throw new PendingThrow("java/lang/IllegalArgumentException"); - if (this.wrap && !this.headerDone) this.dictId = adler32(1, d) >>> 0; - } - private put(...bytes: number[]) { - for (const b of bytes) this.pending[this.pendLen++] = b; - } - private block(last: boolean) { - const n = this.heldLen; - this.put(last ? 1 : 0, n & 0xff, n >> 8, ~n & 0xff, (~n >> 8) & 0xff); - this.pending.set(this.held.subarray(0, n), this.pendLen); - this.pendLen += n; - this.heldLen = 0; - } - private be32(v: number) { - this.put(v >>> 24, (v >>> 16) & 0xff, (v >>> 8) & 0xff, v & 0xff); - } - private drain(output: Uint8Array, at: number): number { - const n = Math.min(output.length - at, this.pendLen - this.pendPos); - output.set(this.pending.subarray(this.pendPos, this.pendPos + n), at); - this.pendPos += n; - if (this.pendPos === this.pendLen) this.pendPos = this.pendLen = 0; - return n; - } - // flush: 0 none, 2 sync, 3 full, 4 finish. Returns [read, written]. - run(input: Uint8Array, output: Uint8Array, flush: number): [number, number] { - if (output.length === 0) return [0, 0]; - let read = 0, written = this.drain(output, 0); - if (this.pendLen) { - this.lastFlush = -1; - return [0, written]; - } - // a repeated flush with no new input adds nothing, so flush loops end - if (input.length === 0 && flush <= this.lastFlush && flush !== 4) return [0, written]; - this.lastFlush = flush; - let flushed = false; - for (;;) { - written += this.drain(output, written); - if (this.pendLen || this.ended) return [read, written]; - if (!this.headerDone) { - this.headerDone = true; - if (this.wrap) { - if (this.dictId === null) this.put(0x78, 0x01); - else { - this.put(0x78, 0x20); - this.be32(this.dictId); - } - } - continue; - } - if (read < input.length) { - const n = Math.min(BLOCK - this.heldLen, input.length - read); - const chunk = input.subarray(read, read + n); - this.held.set(chunk, this.heldLen); - if (this.wrap) this.adler = adler32(this.adler, chunk); - this.heldLen += n; - read += n; - if (this.heldLen === BLOCK) this.block(false); - continue; - } - if (flushed || (flush !== 2 && flush !== 3 && flush !== 4)) return [read, written]; - flushed = true; - if (flush === 4) { - this.block(true); - if (this.wrap) this.be32(this.adler >>> 0); - this.ended = true; - } else { - if (this.heldLen) this.block(false); - this.block(false); - } - } - } -} - function streams(t: JThread): Map { let m = t.vm.state.get("zstreams"); if (!m) t.vm.state.set("zstreams", (m = new Map())); @@ -189,10 +87,14 @@ function setInflateDictionary(t: JThread, addr: bigint, d: Uint8Array) { if (!inflater(t, addr).setDictionary(d)) throw new PendingThrow("java/lang/IllegalArgumentException"); } -function deflate(t: JThread, addr: bigint, input: Uint8Array, output: Uint8Array, flush: number): bigint { +function deflate(t: JThread, addr: bigint, input: Uint8Array, output: Uint8Array, flush: number, params: number): bigint { const z = deflater(t, addr); - const [read, written] = z.run(input, output, flush); - return result(read, written, z.finished, false); + const [read, written, retry] = z.run(input, output, flush, params); + return result(read, written, z.finished, retry); +} + +function setDeflateDictionary(t: JThread, addr: bigint, d: Uint8Array) { + if (!deflater(t, addr).setDictionary(d)) throw new PendingThrow("java/lang/IllegalArgumentException"); } function mem(t: JThread, addr: bigint, len: number) { @@ -221,21 +123,22 @@ export const natives: Natives = { [`${I}reset(J)V`]: (t, [addr]) => inflater(t, addr).reset(), [`${I}end(J)V`]: (t, [addr]) => void streams(t).delete(addr), - [`${D}init(IIZ)J`]: (t, [, , nowrap]) => { + [`${D}init(IIZ)J`]: (t, [level, strategy, nowrap]) => { + if (level < -1 || level > 9 || strategy < 0 || strategy > 2) throw new PendingThrow("java/lang/IllegalArgumentException"); const id = nextStream++; - streams(t).set(id, new Deflate(nowrap === 1)); + streams(t).set(id, new Deflate(level, strategy, nowrap === 1)); return id; }, - [`${D}deflateBytesBytes(J[BII[BIIII)J`]: (t, [, addr, ib, io, il, ob, oo, ol, flush]) => - deflate(t, addr, u8(ib, io, il), u8(ob, oo, ol), flush), - [`${D}deflateBytesBuffer(J[BIIJIII)J`]: (t, [, addr, ib, io, il, oa, ol, flush]) => - deflate(t, addr, u8(ib, io, il), mem(t, oa, ol), flush), - [`${D}deflateBufferBytes(JJI[BIIII)J`]: (t, [, addr, ia, il, ob, oo, ol, flush]) => - deflate(t, addr, mem(t, ia, il), u8(ob, oo, ol), flush), - [`${D}deflateBufferBuffer(JJIJIII)J`]: (t, [, addr, ia, il, oa, ol, flush]) => - deflate(t, addr, mem(t, ia, il), mem(t, oa, ol), flush), - [`${D}setDictionary(J[BII)V`]: (t, [addr, b, off, len]) => deflater(t, addr).setDictionary(u8(b, off, len)), - [`${D}setDictionaryBuffer(JJI)V`]: (t, [addr, a, len]) => deflater(t, addr).setDictionary(mem(t, a, len)), + [`${D}deflateBytesBytes(J[BII[BIIII)J`]: (t, [, addr, ib, io, il, ob, oo, ol, flush, params]) => + deflate(t, addr, u8(ib, io, il), u8(ob, oo, ol), flush, params), + [`${D}deflateBytesBuffer(J[BIIJIII)J`]: (t, [, addr, ib, io, il, oa, ol, flush, params]) => + deflate(t, addr, u8(ib, io, il), mem(t, oa, ol), flush, params), + [`${D}deflateBufferBytes(JJI[BIIII)J`]: (t, [, addr, ia, il, ob, oo, ol, flush, params]) => + deflate(t, addr, mem(t, ia, il), u8(ob, oo, ol), flush, params), + [`${D}deflateBufferBuffer(JJIJIII)J`]: (t, [, addr, ia, il, oa, ol, flush, params]) => + deflate(t, addr, mem(t, ia, il), mem(t, oa, ol), flush, params), + [`${D}setDictionary(J[BII)V`]: (t, [addr, b, off, len]) => setDeflateDictionary(t, addr, u8(b, off, len)), + [`${D}setDictionaryBuffer(JJI)V`]: (t, [addr, a, len]) => setDeflateDictionary(t, addr, mem(t, a, len)), [`${D}getAdler(J)I`]: (t, [addr]) => deflater(t, addr).checksum | 0, [`${D}reset(J)V`]: (t, [addr]) => deflater(t, addr).reset(), [`${D}end(J)V`]: (t, [addr]) => void streams(t).delete(addr), diff --git a/test/deflate.test.ts b/test/deflate.test.ts new file mode 100644 index 0000000..31cd990 --- /dev/null +++ b/test/deflate.test.ts @@ -0,0 +1,279 @@ +// The DEFLATE encoder behind java.util.zip.Deflater: every output must inflate +// with node:zlib and with the VM's own Inflate, and sizes must stay near zlib's. + +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import zlib from "node:zlib"; +import { Deflate } from "../src/deflate.ts"; +import { Inflate } from "../src/inflate.ts"; + +const NO_FLUSH = 0, SYNC = 2, FULL = 3, FINISH = 4; + +function prng(seed: number) { + let s = seed >>> 0 || 1; + return () => { + s ^= s << 13; s >>>= 0; + s ^= s >>> 17; + s ^= s << 5; s >>>= 0; + return s; + }; +} + +function random(n: number, seed: number): Uint8Array { + const r = prng(seed), b = new Uint8Array(n); + for (let i = 0; i < n; i++) b[i] = r(); + return b; +} + +// words from a small vocabulary: compresses like text +function words(n: number, seed: number): Uint8Array { + const r = prng(seed); + const vocab = Array.from({ length: 300 }, () => Array.from({ length: 2 + (r() % 9) }, () => 97 + (r() % 26))); + const b = new Uint8Array(n); + for (let i = 0; i < n;) { + for (const c of vocab[(r() % 300) % (1 + (r() % 300))]) if (i < n) b[i++] = c; + if (i < n) b[i++] = 32; + } + return b; +} + +// short runs and repeats at many distances +function patterned(n: number, seed: number): Uint8Array { + const r = prng(seed), b = new Uint8Array(n); + for (let i = 0; i < n;) { + if (i > 8 && r() % 3) { + const len = 3 + (r() % 300), dist = 1 + (r() % Math.min(i, 40000)); + for (let k = 0; k < len && i < n; k++, i++) b[i] = b[i - dist]; + } else b[i++] = r() % 7 ? r() % 16 : r(); + } + return b; +} + +const srcDir = join(import.meta.dirname, "..", "src"); +const source = Buffer.concat(readdirSync(srcDir).filter((f) => f.endsWith(".ts")).sort().map((f) => readFileSync(join(srcDir, f)))); + +const corpora: Record = { + empty: new Uint8Array(0), + one: Uint8Array.of(65), + zeros: new Uint8Array(200000), + random: random(150000, 1), + words: words(250000, 2), + patterned: patterned(200000, 3), + source: source.subarray(0, 400000), + mixed: Buffer.concat([words(70000, 4), random(40000, 5), new Uint8Array(70000), patterned(50000, 6)]), +}; + +// as Deflater.deflate is driven: input stays put until it is consumed +function drive(d: Deflate, input: Uint8Array, flush: number, outSize: number, parts: Uint8Array[]) { + const out = new Uint8Array(outSize); + let off = 0; + for (;;) { + const [read, written] = d.run(input.subarray(off), out, flush, 0); + off += read; + if (written) parts.push(out.slice(0, written)); + if (flush === FINISH ? d.finished : off === input.length && written < outSize) return; + } +} + +function compress(data: Uint8Array, level: number, strategy = 0, nowrap = false, chunk = data.length || 1, outSize = 4096): Buffer { + const d = new Deflate(level, strategy, nowrap); + const parts: Uint8Array[] = []; + for (let at = 0; at < data.length; at += chunk) drive(d, data.subarray(at, at + chunk), NO_FLUSH, outSize, parts); + drive(d, new Uint8Array(0), FINISH, outSize, parts); + return Buffer.concat(parts); +} + +function vmInflate(z: Uint8Array, nowrap: boolean, dict?: Uint8Array): Buffer { + const inf = new Inflate(nowrap); + if (dict && nowrap) inf.setDictionary(dict); + const out: Buffer[] = []; + const buf = new Uint8Array(65536); + for (let guard = 0; !inf.finished; guard++) { + assert.ok(guard < 100000, "inflate stuck"); + const [read, written] = inf.run(z, buf, 0, buf.length); + if (inf.needDict) assert.ok(inf.setDictionary(dict!)); + out.push(Buffer.from(buf.subarray(0, written))); + assert.ok(read > 0 || written > 0 || inf.finished || inf.needDict); + z = z.subarray(read); + } + return Buffer.concat(out); +} + +function check(z: Buffer, data: Uint8Array, nowrap: boolean) { + const node = nowrap ? zlib.inflateRawSync(z) : zlib.inflateSync(z); + assert.ok(Buffer.from(data).equals(node), "node:zlib output differs"); + assert.ok(Buffer.from(data).equals(vmInflate(z, nowrap)), "Inflate output differs"); +} + +test("every level and strategy round-trips", () => { + for (const [name, data] of Object.entries(corpora)) { + for (let level = 0; level <= 9; level++) { + for (const strategy of [0, 1, 2]) { + if (strategy && level % 3) continue; + const nowrap = (level + strategy) % 2 === 1; + const z = compress(data, level, strategy, nowrap); + try { + check(z, data, nowrap); + } catch (e) { + throw new Error(`${name} level ${level} strategy ${strategy} nowrap ${nowrap}: ${(e as Error).message}`); + } + } + } + } +}); + +test("level -1 is level 6 and the header names the level", () => { + const data = corpora.words; + assert.deepEqual(compress(data, -1), compress(data, 6)); + const header = (level: number, strategy = 0) => [...compress(Uint8Array.of(1, 2, 3), level, strategy).subarray(0, 2)]; + assert.deepEqual(header(0), [0x78, 0x01]); + assert.deepEqual(header(1), [0x78, 0x01]); + assert.deepEqual(header(2), [0x78, 0x5e]); + assert.deepEqual(header(5), [0x78, 0x5e]); + assert.deepEqual(header(6), [0x78, 0x9c]); + assert.deepEqual(header(9), [0x78, 0xda]); + assert.deepEqual(header(9, 2), [0x78, 0x01]); +}); + +test("small output buffers and tiny input chunks", () => { + for (const level of [0, 1, 4, 9]) { + for (const [chunk, outSize] of [[1, 1], [7, 3], [1000, 1], [4099, 17], [70000, 65536]]) { + const data = corpora.mixed.subarray(0, 30000); + check(compress(data, level, 0, false, chunk, outSize), data, false); + } + } +}); + +test("input larger than the window slides it", () => { + const data = corpora.words; + for (const level of [1, 3, 6, 9]) check(compress(data, level, 0, true, 65536 + 17), data, true); +}); + +test("sync flush ends on a byte boundary with an empty stored block", () => { + for (const level of [0, 1, 6, 9]) { + const d = new Deflate(level, 0, true); + const parts: Uint8Array[] = []; + const data = corpora.patterned; + for (let at = 0; at < 60000; at += 5000) { + drive(d, data.subarray(at, at + 5000), SYNC, 300, parts); + const sofar = Buffer.concat(parts); + assert.deepEqual([...sofar.subarray(-4)], [0, 0, 0xff, 0xff]); + // everything fed so far can be decoded without the rest of the stream + const got = zlib.inflateRawSync(sofar, { finishFlush: zlib.constants.Z_SYNC_FLUSH }); + assert.ok(Buffer.from(data.subarray(0, at + 5000)).equals(got), `level ${level} at ${at}`); + } + drive(d, new Uint8Array(0), FINISH, 300, parts); + check(Buffer.concat(parts), data.subarray(0, 60000), true); + } +}); + +test("a repeated flush with no input adds nothing", () => { + const d = new Deflate(6, 0, false); + const parts: Uint8Array[] = []; + drive(d, corpora.words.subarray(0, 1000), SYNC, 64, parts); + const out = new Uint8Array(64); + assert.equal(d.run(new Uint8Array(0), out, SYNC, 0)[1], 0); +}); + +test("full flush resets the match history", () => { + for (const level of [1, 6, 9]) { + const a = corpora.words.subarray(0, 20000); + const d = new Deflate(level, 0, true); + const first: Uint8Array[] = [], second: Uint8Array[] = []; + drive(d, a, FULL, 1000, first); + drive(d, a, NO_FLUSH, 1000, second); + drive(d, new Uint8Array(0), FINISH, 1000, second); + // the same data again would match the first copy if history were kept + const tail = Buffer.concat(second); + assert.ok(Buffer.from(a).equals(zlib.inflateRawSync(tail)), `level ${level}: second part stands alone`); + check(Buffer.concat([...first, ...second]), Buffer.concat([a, a]), true); + } +}); + +test("a preset dictionary is used and named in the header", () => { + const dict = corpora.words.subarray(0, 20000); + const data = corpora.words.subarray(5000, 15000); + for (const level of [1, 6, 9]) { + for (const nowrap of [false, true]) { + const d = new Deflate(level, 0, nowrap); + assert.ok(d.setDictionary(dict)); + const parts: Uint8Array[] = []; + drive(d, data, NO_FLUSH, 4096, parts); + drive(d, new Uint8Array(0), FINISH, 4096, parts); + const z = Buffer.concat(parts); + const plain = compress(data, level, 0, nowrap).length; + assert.ok(z.length < plain / 4, `level ${level}: ${z.length} vs ${plain}`); + const node = nowrap ? zlib.inflateRawSync(z, { dictionary: dict }) : zlib.inflateSync(z, { dictionary: dict }); + assert.ok(Buffer.from(data).equals(node)); + assert.ok(Buffer.from(data).equals(vmInflate(z, nowrap, dict))); + if (!nowrap) assert.equal(z.readUInt32BE(2) >>> 0, adler(dict)); + } + } +}); + +function adler(d: Uint8Array): number { + let s1 = 1, s2 = 0; + for (const b of d) { + s1 = (s1 + b) % 65521; + s2 = (s2 + s1) % 65521; + } + return ((s2 << 16) | s1) >>> 0; +} + +test("a dictionary is refused once a zlib stream has started", () => { + const d = new Deflate(6, 0, false); + assert.ok(d.setDictionary(Uint8Array.of(1, 2, 3))); + d.run(Uint8Array.of(1, 2, 3), new Uint8Array(100), NO_FLUSH, 0); + assert.equal(d.setDictionary(Uint8Array.of(1, 2, 3)), false); +}); + +test("changing level and strategy mid-stream", () => { + const data = corpora.mixed; + const steps: [number, number][] = [[1, 0], [9, 0], [0, 0], [6, 1], [4, 2], [3, 0], [6, 0]]; + for (const outSize of [16, 4096]) { + const d = new Deflate(6, 0, false); + const parts: Uint8Array[] = []; + const out = new Uint8Array(outSize); + const size = Math.floor(data.length / steps.length); + steps.forEach(([level, strategy], i) => { + let input = data.subarray(i * size, i === steps.length - 1 ? data.length : (i + 1) * size); + const params = 1 | (strategy << 1) | (level << 3); + let first = true; + // as Deflater: params go with each call until one reports it is done + for (;;) { + const [read, written, retry] = d.run(input, out, NO_FLUSH, first ? params : 0); + input = input.subarray(read); + if (written) parts.push(out.slice(0, written)); + if (retry) { + first = true; + continue; + } + first = false; + const [r, w] = d.run(input, out, NO_FLUSH, 0); + input = input.subarray(r); + if (w) parts.push(out.slice(0, w)); + if (input.length === 0 && w < outSize) break; + } + }); + drive(d, new Uint8Array(0), FINISH, outSize, parts); + check(Buffer.concat(parts), data, false); + } +}); + +test("compressed size is near zlib's", () => { + for (const name of ["words", "source", "patterned", "mixed"]) { + const data = corpora[name]; + for (const [level, slack] of [[1, 1.25], [6, 1.1], [9, 1.1]] as const) { + const ours = compress(data, level).length; + const theirs = zlib.deflateSync(data, { level }).length; + assert.ok(ours <= theirs * slack, `${name} level ${level}: ${ours} vs zlib ${theirs}`); + } + assert.ok(compress(data, 9).length <= compress(data, 1).length); + } + const stored = compress(corpora.random, 6); + assert.ok(stored.length <= corpora.random.length + 5 * Math.ceil(corpora.random.length / 16383) + 12, `incompressible data costs a few bytes: ${stored.length}`); + assert.ok(compress(corpora.zeros, 6).length < 400); + assert.ok(compress(corpora.words, 6, 2).length < corpora.words.length, "huffman only still codes"); +}); diff --git a/test/java/DeflaterLevels.java b/test/java/DeflaterLevels.java new file mode 100644 index 0000000..f7a30d5 --- /dev/null +++ b/test/java/DeflaterLevels.java @@ -0,0 +1,274 @@ +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.Arrays; +import java.util.Random; +import java.util.zip.DataFormatException; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import java.util.zip.Inflater; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +public class DeflaterLevels { + static final String[] WORDS = {"alpha", "beta", "gamma", "delta", "class", "public", "static", "void", "return", "String"}; + + static byte[] text(int n, long seed) { + Random r = new Random(seed); + StringBuilder sb = new StringBuilder(); + while (sb.length() < n) sb.append(WORDS[r.nextInt(WORDS.length)]).append(r.nextInt(5) == 0 ? "\n" : " "); + return Arrays.copyOf(sb.toString().getBytes(), n); + } + + static byte[] noise(int n, long seed) { + byte[] b = new byte[n]; + new Random(seed).nextBytes(b); + return b; + } + + static byte[] inflate(byte[] z, boolean nowrap, byte[] dict, int size) throws DataFormatException { + Inflater inf = new Inflater(nowrap); + inf.setInput(z); + byte[] out = new byte[size + 10]; + int n = 0; + while (!inf.finished()) { + int k = inf.inflate(out, n, out.length - n); + if (k == 0 && inf.needsDictionary()) inf.setDictionary(dict); + else if (k == 0 && (inf.needsInput() || n == out.length)) break; + n += k; + } + boolean done = inf.finished(); + inf.end(); + return done ? Arrays.copyOf(out, n) : null; + } + + static byte[] compress(byte[] data, int level, int strategy, boolean nowrap, int step, int outSize) { + Deflater d = new Deflater(level, nowrap); + d.setStrategy(strategy); + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + byte[] buf = new byte[outSize]; + for (int at = 0; at < data.length; at += step) { + d.setInput(data, at, Math.min(step, data.length - at)); + while (!d.needsInput()) bo.write(buf, 0, d.deflate(buf)); + } + d.finish(); + while (!d.finished()) bo.write(buf, 0, d.deflate(buf)); + d.end(); + return bo.toByteArray(); + } + + static void levels() throws Exception { + byte[][] inputs = {new byte[0], {7}, text(100000, 1), noise(70000, 2), new byte[150000], text(30000, 3)}; + String[] names = {"empty", "one", "text", "noise", "zeros", "short"}; + for (int i = 0; i < inputs.length; i++) { + for (int level = -1; level <= 9; level++) { + for (int strategy = 0; strategy <= 2; strategy++) { + boolean nowrap = (level + strategy) % 2 != 0; + byte[] z = compress(inputs[i], level, strategy, nowrap, 1 << 20, 4096); + byte[] back = inflate(z, nowrap, null, inputs[i].length); + boolean ok = back != null && Arrays.equals(back, inputs[i]); + boolean small = level != 0 && inputs[i].length > 1000 && i != 3 && z.length < inputs[i].length; + if (!ok || (level != 0 && i != 3 && inputs[i].length > 1000 && !small)) { + System.out.println("FAIL " + names[i] + " level " + level + " strategy " + strategy); + } + } + } + System.out.println(names[i] + ": all levels and strategies round-trip"); + } + byte[] data = text(200000, 4); + System.out.println("small buffers: " + Arrays.equals(inflate(compress(data, 6, 0, false, 777, 5), false, null, data.length), data)); + System.out.println("level 0 header: " + hex(compress(data, 0, 0, false, 1 << 20, 4096), 2)); + System.out.println("level 1 header: " + hex(compress(data, 1, 0, false, 1 << 20, 4096), 2)); + System.out.println("level 4 header: " + hex(compress(data, 4, 0, false, 1 << 20, 4096), 2)); + System.out.println("level 6 header: " + hex(compress(data, 6, 0, false, 1 << 20, 4096), 2)); + System.out.println("level 9 header: " + hex(compress(data, 9, 0, false, 1 << 20, 4096), 2)); + System.out.println("huffman only header: " + hex(compress(data, 9, Deflater.HUFFMAN_ONLY, false, 1 << 20, 4096), 2)); + byte[] big = compress(data, 6, 0, false, 1 << 20, 4096); + System.out.println("level 6 shrinks text: " + (big.length < data.length / 2)); + System.out.println("huffman only shrinks text: " + + (compress(data, 6, Deflater.HUFFMAN_ONLY, true, 1 << 20, 4096).length < data.length * 3 / 4)); + } + + static String hex(byte[] b, int n) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < n && i < b.length; i++) sb.append(String.format("%02x", b[i] & 0xff)); + return sb.toString(); + } + + static void flushes() throws Exception { + byte[] data = text(60000, 5); + for (int level : new int[] {0, 1, 6, 9}) { + for (int mode : new int[] {Deflater.SYNC_FLUSH, Deflater.FULL_FLUSH}) { + Deflater d = new Deflater(level, true); + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + byte[] buf = new byte[100]; + boolean markers = true; + for (int at = 0; at < data.length; at += 7000) { + d.setInput(data, at, Math.min(7000, data.length - at)); + int k; + while ((k = d.deflate(buf, 0, buf.length, mode)) > 0) bo.write(buf, 0, k); + byte[] sofar = bo.toByteArray(); + int n = sofar.length; + markers &= n >= 4 && sofar[n - 4] == 0 && sofar[n - 3] == 0 && sofar[n - 2] == -1 && sofar[n - 1] == -1; + Inflater inf = new Inflater(true); + inf.setInput(sofar); + byte[] back = new byte[data.length]; + int got = 0, m; + while ((m = inf.inflate(back, got, back.length - got)) > 0) got += m; + markers &= got == Math.min(at + 7000, data.length) && Arrays.equals(Arrays.copyOf(back, got), Arrays.copyOf(data, got)); + inf.end(); + } + d.finish(); + int k; + while (!d.finished()) { + k = d.deflate(buf); + bo.write(buf, 0, k); + } + d.end(); + boolean ok = Arrays.equals(inflate(bo.toByteArray(), true, null, data.length), data); + System.out.println("level " + level + " flush " + mode + ": markers " + markers + ", round-trip " + ok); + } + } + } + + static void params() throws Exception { + byte[] data = text(120000, 6); + Deflater d = new Deflater(Deflater.BEST_SPEED); + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + byte[] buf = new byte[300]; + int[] levels = {1, 9, 0, 6, 3}; + int[] strategies = {0, 0, 0, Deflater.FILTERED, Deflater.HUFFMAN_ONLY}; + int part = data.length / levels.length; + for (int i = 0; i < levels.length; i++) { + d.setLevel(levels[i]); + d.setStrategy(strategies[i]); + d.setInput(data, i * part, part); + while (!d.needsInput()) bo.write(buf, 0, d.deflate(buf)); + } + d.finish(); + while (!d.finished()) bo.write(buf, 0, d.deflate(buf)); + long in = d.getBytesRead(); + d.end(); + System.out.println("parameter changes: read " + in + ", round-trip " + + Arrays.equals(inflate(bo.toByteArray(), false, null, data.length), Arrays.copyOf(data, part * levels.length))); + } + + static void dictionary() throws Exception { + byte[] dict = text(20000, 7); + byte[] data = Arrays.copyOfRange(dict, 5000, 15000); + for (boolean nowrap : new boolean[] {false, true}) { + Deflater d = new Deflater(6, nowrap); + d.setDictionary(dict); + d.setInput(data); + d.finish(); + byte[] buf = new byte[1000]; + int n = d.deflate(buf); + byte[] z = Arrays.copyOf(buf, n); + Inflater inf = new Inflater(nowrap); + inf.setInput(z); + byte[] back = new byte[data.length]; + if (nowrap) inf.setDictionary(dict); + int got = inf.inflate(back); + if (!nowrap && got == 0 && inf.needsDictionary()) { + inf.setDictionary(dict); + got = inf.inflate(back); + } + System.out.println("dictionary nowrap=" + nowrap + ": finished " + d.finished() + ", uses it " + (n < 200) + + ", round-trip " + (got == data.length && Arrays.equals(back, data))); + d.end(); + inf.end(); + } + } + + static void streams() throws Exception { + byte[] data = text(250000, 8); + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + try (GZIPOutputStream g = new GZIPOutputStream(bo)) { + g.write(data, 0, 100000); + g.write(data, 100000, 150000); + } + byte[] gz = bo.toByteArray(); + try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(gz))) { + System.out.println("gzip: header " + hex(gz, 3) + ", round-trip " + Arrays.equals(in.readAllBytes(), data) + + ", shrinks " + (gz.length < data.length / 2)); + } + bo.reset(); + try (GZIPOutputStream g = new GZIPOutputStream(bo, 512, true)) { + g.write(data, 0, 5000); + g.flush(); + System.out.println("gzip flush emits data: " + (bo.size() > 10)); + g.write(data, 5000, 5000); + } + try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(bo.toByteArray()))) { + System.out.println("gzip sync flush: " + Arrays.equals(in.readAllBytes(), Arrays.copyOf(data, 10000))); + } + bo.reset(); + Deflater fast = new Deflater(Deflater.BEST_COMPRESSION); + try (DeflaterOutputStream out = new DeflaterOutputStream(bo, fast)) { + for (int i = 0; i < data.length; i += 1000) out.write(data, i, Math.min(1000, data.length - i)); + } + fast.end(); + System.out.println("deflater stream: " + Arrays.equals(inflate(bo.toByteArray(), false, null, data.length), data)); + + bo.reset(); + try (ZipOutputStream zo = new ZipOutputStream(bo)) { + for (int i = 0; i < 3; i++) { + ZipEntry e = new ZipEntry("e" + i + ".txt"); + zo.putNextEntry(e); + zo.write(data, i * 1000, 90000); + zo.closeEntry(); + System.out.println("zip entry " + i + ": size " + e.getSize() + ", smaller " + (e.getCompressedSize() < e.getSize())); + } + } + try (ZipInputStream zi = new ZipInputStream(new ByteArrayInputStream(bo.toByteArray()))) { + ZipEntry e; + int i = 0; + while ((e = zi.getNextEntry()) != null) { + System.out.println(e.getName() + ": " + Arrays.equals(zi.readAllBytes(), Arrays.copyOfRange(data, i * 1000, i * 1000 + 90000))); + i++; + } + } + } + + static void firstCall() { + Deflater d = new Deflater(); + byte[] buf = new byte[64]; + System.out.println("first call, no input: " + d.deflate(buf)); + System.out.println("needsInput: " + d.needsInput()); + d.setInput(new byte[] {1, 2, 3, 4}); + System.out.println("with input, no flush: " + d.deflate(buf)); + d.finish(); + int n = d.deflate(buf); + System.out.println("finish: " + n + " " + d.finished() + " " + hex(Arrays.copyOfRange(buf, 0, n), 2)); + d.reset(); + d.setInput(new byte[0]); + d.finish(); + n = d.deflate(buf); + System.out.println("empty stream: " + n + " " + hex(buf, n) + " " + d.finished()); + d.end(); + Deflater raw = new Deflater(Deflater.DEFAULT_COMPRESSION, true); + raw.finish(); + n = raw.deflate(buf); + System.out.println("empty raw stream: " + n + " " + hex(buf, n)); + raw.end(); + for (int bad : new int[] {10, -2}) { + try { + new Deflater(bad); + System.out.println("level " + bad + " accepted"); + } catch (IllegalArgumentException e) { + System.out.println("level " + bad + ": IllegalArgumentException"); + } + } + } + + public static void main(String[] args) throws Exception { + levels(); + flushes(); + params(); + dictionary(); + streams(); + firstCall(); + } +} diff --git a/test/jtreg/passing.txt b/test/jtreg/passing.txt index 236aeb8..512c39a 100644 --- a/test/jtreg/passing.txt +++ b/test/jtreg/passing.txt @@ -3368,6 +3368,7 @@ java/util/zip/Bounds.java java/util/zip/CloseInflaterDeflaterTest.java java/util/zip/ConstructDeflaterInput.java java/util/zip/ConstructInflaterOutput.java +java/util/zip/CopyZipFile.java java/util/zip/DataDescriptor.java java/util/zip/DataDescriptorIgnoreCrcAndSizeFields.java java/util/zip/DataDescriptorSignatureMissing.java diff --git a/test/jtreg/times.tsv b/test/jtreg/times.tsv index 982a3b8..0cc43e7 100644 --- a/test/jtreg/times.tsv +++ b/test/jtreg/times.tsv @@ -640,6 +640,7 @@ 8380 java/lang/Thread/BuilderTest.java 8373 java/util/Scanner/EncodingTest.java 8372 java/beans/PropertyEditor/TestByteClassNull.java +8353 java/util/zip/CopyZipFile.java 8352 java/security/cert/CertPathBuilder/selfIssued/DisableRevocation.java 8349 java/nio/channels/Selector/Alias.java 8341 java/net/Socket/DeadlockTest.java