--- a/go.mod +++ b/go.mod @@ -301,7 +301,7 @@ github.com/jessevdk/go-flags v1.6.1 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect - github.com/klauspost/compress v1.19.0 + github.com/klauspost/compress v1.20.2 github.com/klauspost/cpuid/v2 v2.2.11 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.15 // indirect --- a/go.sum +++ b/go.sum @@ -715,8 +715,8 @@ github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= -github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo= +github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -6,7 +6,9 @@ * [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy. * Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). * [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams. +* [lzw](https://github.com/klauspost/compress/tree/master/lzw) is a drop-in replacement for `compress/lzw` with 1.4-4x faster decompression and 1.1-2.7x faster compression, depending on the data. * [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. +* [Xpress](https://github.com/klauspost/compress/tree/master/xpress) decompression of the Microsoft XPRESS (MS-XCA) plain LZ77 and LZ77+Huffman formats (the LZ77+Huffman variant is the one used in WIM images and Windows Compact OS / WOF data). * [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently. * [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. @@ -27,6 +29,18 @@ # changelog +* Jul 1st, 2026 [1.19.0](https://github.com/klauspost/compress/releases/tag/v1.19.0) + * zstd: Add true concurrent stream encodingin https://github.com/klauspost/compress/pull/1136 + * zstd: arm64 decoder asm by @lizthegrey in https://github.com/klauspost/compress/pull/1160 + * flate: Add inflate checkpoints in https://github.com/klauspost/compress/pull/1154 + * zstd: avoid unused BuildDict encoder allocation by @snissn in https://github.com/klauspost/compress/pull/1147 + * snappy/s2: Limit length of varint in `decodedLen` by @eustas in https://github.com/klauspost/compress/pull/1148 + * gzhttp: match qvalue parameter case-insensitively (RFC 7231) by @z9z in https://github.com/klauspost/compress/pull/1149 + * zip: add NameDecoder callback for legacy encoding rewrite by @SAY-5 in https://github.com/klauspost/compress/pull/1150 + * huff0: Allow building tables from histogram in https://github.com/klauspost/compress/pull/1155 + * huff0: Allow building table from oversized histogram in https://github.com/klauspost/compress/pull/1156 + * s2sx: Clean symlink targets in https://github.com/klauspost/compress/pull/1163 + * Feb 9th, 2026 [1.18.4](https://github.com/klauspost/compress/releases/tag/v1.18.4) * gzhttp: Add zstandard to server handler wrapper https://github.com/klauspost/compress/pull/1121 * zstd: Add ResetWithOptions to encoder/decoder https://github.com/klauspost/compress/pull/1122 --- a/vendor/github.com/klauspost/compress/flate/deflate.go +++ b/vendor/github.com/klauspost/compress/flate/deflate.go @@ -95,6 +95,11 @@ index int hashMatch [maxMatchLength + minMatchLength]uint32 + // litBits[i] is the number of bits needed to encode window[index:i] + // as literals, where index is the window index at the start of the + // current deflateLazy call. Only allocated when d.chain > 100 (level 9). + litBits []int32 + // Input hash chains // hashHead[hashValue] contains the largest inputIndex with the specified hash value // If hashHead[hashValue] is within the current window, then @@ -260,6 +265,7 @@ } // Update window information. d.windowEnd += n + d.blockStart = d.windowEnd s.index = n } @@ -283,7 +289,9 @@ minIndex := max(pos-windowSize, 0) offset = 0 - if d.chain < 100 { + // The gain estimate below needs the tables that deflateLazy only + // generates when d.chain > 100, so the conditions must match. + if d.chain <= 100 { for i := prevHead; tries > 0; tries-- { if wEnd == win[i+length] { n := matchLen(win[i:i+minMatchLook], wPos) @@ -311,21 +319,25 @@ } // Minimum gain to accept a match. - cGain := 4 + cGain := int32(4) // Some like it higher (CSV), some like it lower (JSON) const baseCost = 3 // Base is 4 bytes at with an additional cost. // Matches must be better than this. + s := d.state + hashPrev := &s.hashPrev + hashOffset := s.hashOffset + litBits := s.litBits[pos : pos+minMatchLook+1] + for i := prevHead; tries > 0; tries-- { if wEnd == win[i+length] { n := matchLen(win[i:i+minMatchLook], wPos) if n > length { // Calculate gain. Estimate - newGain := d.h.bitLengthRaw(wPos[:n]) - int(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int(lengthExtraBits[lengthCodes[(n-3)&255]]) + newGain := litBits[n] - litBits[0] - int32(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int32(lengthExtraBits[lengthCodes[(n-3)&255]]) - //fmt.Println("gain:", newGain, "prev:", cGain, "raw:", d.h.bitLengthRaw(wPos[:n]), "this-len:", n, "prev-len:", length) if newGain > cGain { length = n offset = pos - i @@ -343,7 +355,7 @@ // hashPrev[i & windowMask] has already been overwritten, so stop now. break } - i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset + i = int(hashPrev[i&windowMask]) - hashOffset if i < minIndex { break } @@ -427,6 +439,20 @@ tmp[v]++ } d.h.generate(tmp[:], 15) + + // Compute the cumulative cost of encoding the window as literals, + // so findMatch can look up the cost of any range in constant time. + if s.litBits == nil { + s.litBits = make([]int32, 2*windowSize+1) + } + codes := d.h.codes[:256] + litBits := s.litBits[s.index : d.windowEnd+1] + total := int32(0) + litBits[0] = 0 + for i, v := range d.window[s.index:d.windowEnd] { + total += int32(codes[v].len()) + litBits[i+1] = total + } } s.maxInsertIndex = d.windowEnd - (minMatchLength - 1) --- a/vendor/github.com/klauspost/compress/flate/fast_encoder.go +++ b/vendor/github.com/klauspost/compress/flate/fast_encoder.go @@ -9,6 +9,7 @@ "fmt" "github.com/klauspost/compress/internal/le" + "github.com/klauspost/compress/internal/regmask" ) type fastEnc interface { @@ -108,7 +109,7 @@ // hash7 returns the hash of the lowest 7 bytes of u to fit in a hash table with h bits. // Preferably h should be a constant and should always be <64. func hash7(u uint64, h uint8) uint32 { - return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & reg8SizeMask64)) + return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & regmask.Shift64ByUint8)) } // hashLen returns a hash of the lowest mls bytes of with length output bits. --- a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go +++ b/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go @@ -32,14 +32,12 @@ // bufferFlushSize indicates the buffer size // after which bytes are flushed to the writer. - // Should preferably be a multiple of 6, since - // we accumulate 6 bytes between writes to the buffer. + // Between checks, at most 7 bytes are added to the buffer, + // and writes are done as unconditional 8-byte stores, so the + // buffer must have at least bufferFlushSize+7+8 bytes. bufferFlushSize = 246 ) -// Minimum length code that emits bits. -const lengthExtraBitsMinCode = 8 - // The number of extra bits needed by length code X - LENGTH_CODES_START. var lengthExtraBits = [32]uint8{ /* 257 */ 0, 0, 0, @@ -55,9 +53,6 @@ 64, 80, 96, 112, 128, 160, 192, 224, 255, } -// Minimum offset code that emits bits. -const offsetExtraBitsMinCode = 4 - // offset code word extra bits. var offsetExtraBits = [32]int8{ 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, @@ -67,8 +62,24 @@ 14, 14, } +// offsetCombined combines the number of extra bits and the base offset +// of each offset code in a single table: extra bits in the low 8 bits, +// and the base offset (already reduced by baseMatchOffset) in the upper bits. +// Entries for the extended window codes 30 and 31 are unused. var offsetCombined = [32]uint32{} +// lengthCombined combines, for each match length (reduced by +// baseMatchLength), the length code (relative to lengthCodesStart) in +// bits 0-4, the number of extra bits in bits 5-7 and the value of the +// extra bits in bits 8-12. +var lengthCombined = func() (t [256]uint32) { + for i := range t { + code := lengthCodes[i] + t[i] = uint32(code) | uint32(lengthExtraBits[code])<<5 | uint32(uint8(i)-lengthBase[code])<<8 + } + return t +}() + func init() { var offsetBase = [32]uint32{ /* normal deflate */ @@ -85,7 +96,7 @@ for i := range offsetCombined[:] { // Don't use extended window values... - if offsetExtraBits[i] == 0 || offsetBase[i] > 0x006000 { + if offsetBase[i] > 0x006000 { continue } offsetCombined[i] = uint32(offsetExtraBits[i]) | (offsetBase[i] << 8) @@ -115,10 +126,14 @@ lastHeader int // Set between 0 (reused block can be up to 2x the size) logNewTablePenalty uint - bytes [256 + 8]byte - literalFreq [lengthCodesStart + 32]uint16 - offsetFreq [32]uint16 - codegenFreq [codegenCodeCount]uint16 + + // bytes must hold at least bufferFlushSize+7+8 bytes (see bufferFlushSize). + // Its size is rounded up to a multiple of 8 to keep the following + // fields aligned, which is measurably faster in writeBlockHuff. + bytes [(bufferFlushSize + 7 + 8 + 7) &^ 7]byte + literalFreq [lengthCodesStart + 32]uint16 + offsetFreq [32]uint16 + codegenFreq [codegenCodeCount]uint16 // codegen must have an extra space for the final symbol. codegen [literalCount + offsetCodeCount + 1]uint8 @@ -842,124 +857,75 @@ lengths := leCodes[lengthCodesStart:] lengths = lengths[:32] - // Go 1.16 LOVES having these on stack. + // Keeping these on the stack instead of in w is significantly faster. bits, nbits, nbytes := w.bits, w.nbits, w.nbytes - for _, t := range tokens { - if t < 256 { - //w.writeCode(lits[t.literal()]) - c := lits[t] - bits |= c.code64() << (nbits & 63) - nbits += c.len() - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } - } - continue + // Flush whole bytes, so that nbits <= 7 when entering the loop below. + le.Store64(w.bytes[:], nbytes, bits) + nbytes += nbits >> 3 + bits >>= nbits & 56 + nbits &= 7 + if nbytes >= bufferFlushSize { + _, w.err = w.writer.Write(w.bytes[:nbytes]) + nbytes = 0 + if w.err != nil { + return } + } - // Write the length - length := t.length() - lengthCode := lengthCode(length) & 31 - if false { - w.writeCode(lengths[lengthCode]) - } else { - // inlined - c := lengths[lengthCode] + // The loop below flushes whole bytes at the end of every iteration, + // so that nbits <= 7 at the top of every iteration. + // A literal adds at most 15 bits and a match at most + // 15+5+15+13 = 48 bits, so up to three literals or a single match + // can be added to the 64-bit accumulator without an intermediate flush. + // Unconditionally storing 8 bytes and advancing by the number of + // whole bytes avoids a poorly predicted branch per token. + for i := 0; i < len(tokens); i++ { + t := tokens[i] + if t < 256 { + c := lits[t] bits |= c.code64() << (nbits & 63) nbits += c.len() - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 + // Add up to two more literals before flushing. + if i+1 < len(tokens) && tokens[i+1] < 256 { + i++ + c := lits[tokens[i]] + bits |= c.code64() << (nbits & 63) + nbits += c.len() + if i+1 < len(tokens) && tokens[i+1] < 256 { + i++ + c := lits[tokens[i]] + bits |= c.code64() << (nbits & 63) + nbits += c.len() } } - } - - if lengthCode >= lengthExtraBitsMinCode { - extraLengthBits := lengthExtraBits[lengthCode] - //w.writeBits(extraLength, extraLengthBits) - extraLength := int32(length - lengthBase[lengthCode]) - bits |= uint64(extraLength) << (nbits & 63) - nbits += extraLengthBits - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } - } - } - // Write the offset - offset := t.offset() - offsetCode := (offset >> 16) & 31 - if false { - w.writeCode(offs[offsetCode]) } else { - // inlined - c := offs[offsetCode] - bits |= c.code64() << (nbits & 63) - nbits += c.len() - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } - } - } - - if offsetCode >= offsetExtraBitsMinCode { - offsetComb := offsetCombined[offsetCode] - //w.writeBits(extraOffset, extraOffsetBits) - bits |= uint64((offset-(offsetComb>>8))&matchOffsetOnlyMask) << (nbits & 63) - nbits += uint8(offsetComb) - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } + // Write the length code and its extra bits as one unit. + lc := lengthCombined[t.length()] + c := lengths[lc&31] + bits |= (c.code64() | uint64(lc>>8)<<(c.len()&63)) << (nbits & 63) + nbits += c.len() + uint8(lc>>5)&7 + + // Write the offset code and its extra bits as one unit. + offset := t.offset() + offCode := (offset >> 16) & 31 + c = offs[offCode] + offComb := offsetCombined[offCode] + extra := (offset - (offComb >> 8)) & matchOffsetOnlyMask + bits |= (c.code64() | uint64(extra)<<(c.len()&63)) << (nbits & 63) + nbits += c.len() + uint8(offComb) + } + le.Store64(w.bytes[:], nbytes, bits) + nbytes += nbits >> 3 + bits >>= nbits & 56 + nbits &= 7 + if nbytes >= bufferFlushSize { + if w.err != nil { + nbytes = 0 + return } + _, w.err = w.writer.Write(w.bytes[:nbytes]) + nbytes = 0 } } // Restore... @@ -1094,8 +1060,8 @@ if debugDeflate { count -= int(nbytes)*8 + int(nbits) } - // Unroll, write 3 codes/loop. - // Fastest number of unrolls. + // Write 3 codes per iteration. The three codes are combined first, + // independently of the accumulator, and merged with a single shift. for len(input) > 3 { // We must have at least 48 bits free. if nbits >= 8 { @@ -1116,13 +1082,13 @@ _, w.err = w.writer.Write(w.bytes[:nbytes]) nbytes = 0 } - a, b := encoding[input[0]], encoding[input[1]] - bits |= a.code64() << (nbits & 63) - bits |= b.code64() << ((nbits + a.len()) & 63) - c := encoding[input[2]] - nbits += b.len() + a.len() - bits |= c.code64() << (nbits & 63) - nbits += c.len() + a, b, c := encoding[input[0]], encoding[input[1]], encoding[input[2]] + v := a.code64() | b.code64()<<(a.len()&63) + n := a.len() + b.len() + v |= c.code64() << (n & 63) + n += c.len() + bits |= v << (nbits & 63) + nbits += n input = input[3:] } --- a/vendor/github.com/klauspost/compress/flate/huffman_code.go +++ b/vendor/github.com/klauspost/compress/flate/huffman_code.go @@ -38,6 +38,9 @@ // Possible lengths are codegenCodeCount, offsetCodeCount and literalCount. // The largest of these is literalCount, so we allocate for that case. freqcache [literalCount + 1]literalNode + + // sortcache is scratch space for sortByFreq. + sortcache [literalCount + 1]literalNode } type literalNode struct { @@ -139,14 +142,6 @@ return total } -func (h *huffmanEncoder) bitLengthRaw(b []byte) int { - var total int - for _, f := range b { - total += int(h.codes[f].len()) - } - return total -} - // canReuseBits returns the number of bits or math.MaxInt32 if the encoder cannot be reused. func (h *huffmanEncoder) canReuseBits(freq []uint16) int { var total int @@ -309,27 +304,93 @@ } // Look at the leaves and assign them a bit count and an encoding as specified -// in RFC 1951 3.2.2 -func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) { - code := uint16(0) +// in RFC 1951 3.2.2. +// list must be sorted by increasing frequency and codes must have +// all entries for literals not in list set to zero. +func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode, codes []hcode) { + // The literals list[len(list)-bits] .. list[len(list)-1] + // are encoded using "bits" bits. Record the length of each literal + // temporarily in codes. for n, bits := range bitCount { - code <<= 1 if n == 0 || bits == 0 { continue } - // The literals list[len(list)-bits] .. list[len(list)-bits] - // are encoded using "bits" bits, and get the values - // code, code + 1, .... The code values are - // assigned in literal order (not frequency order). - chunk := list[len(list)-int(bits):] - - sortByLiteral(chunk) - for _, node := range chunk { - h.codes[node.literal] = newhcode(reverseBits(code, uint8(n)), uint8(n)) - code++ + for _, node := range list[len(list)-int(bits):] { + codes[node.literal] = hcode(n) } list = list[0 : len(list)-int(bits)] } + + // Code values are assigned in literal order (not frequency order) + // within each code length, starting from the first code of each length. + var nextCode [maxBitsLimit]uint16 + code := uint16(0) + for n := 1; n < len(bitCount); n++ { + code = (code + uint16(bitCount[n-1])) << 1 + nextCode[n] = code + } + for i, c := range codes { + if c == 0 { + continue + } + // n is at most maxBitsLimit-1; the mask only elides bounds checks. + n := uint8(c) + codes[i] = newhcode(reverseBits(nextCode[n&15], n), n) + nextCode[n&15]++ + } +} + +// sortByFreq sorts list by increasing frequency. +// The sort is stable, so nodes with equal frequencies keep their order. +func (h *huffmanEncoder) sortByFreq(list []literalNode) { + if len(list) <= 32 { + // Insertion sort is faster for small lists. + for i := 1; i < len(list); i++ { + n := list[i] + j := i + for ; j > 0 && list[j-1].freq > n.freq; j-- { + list[j] = list[j-1] + } + list[j] = n + } + return + } + + // Counting sort on the low byte and then on the high byte + // of the frequency. Both passes are stable. + tmp := h.sortcache[:len(list)] + var count [256]uint16 + var maxFreq uint16 + for _, n := range list { + count[n.freq&0xff]++ + maxFreq |= n.freq + } + var pos uint16 + for i, c := range count { + count[i] = pos + pos += c + } + for _, n := range list { + tmp[count[n.freq&0xff]] = n + count[n.freq&0xff]++ + } + if maxFreq < 256 { + copy(list, tmp) + return + } + count = [256]uint16{} + for _, n := range tmp { + count[n.freq>>8]++ + } + pos = 0 + for i, c := range count { + count[i] = pos + pos += c + } + for _, n := range tmp { + list[count[n.freq>>8]] = n + count[n.freq>>8]++ + } } // Update this Huffman Code object to be the minimum code for the specified frequency count. @@ -362,12 +423,14 @@ } return } - sortByFreq(list) + // list is in order of increasing literal value, so a stable sort + // by frequency orders it by (frequency, literal). + h.sortByFreq(list) // Get the number of literals for each bit count bitCount := h.bitCounts(list, maxBits) // And do the assignment - h.assignEncodingAndSize(bitCount, list) + h.assignEncodingAndSize(bitCount, list, codes) } // atLeastOne clamps the result between 1 and 15. @@ -394,24 +457,27 @@ } func histogramSplit(b []byte, h []uint16) { - // Tested, and slightly faster than 2-way. - // Writing to separate arrays and combining is also slightly slower. - h = h[:256] + // Walk four quarters in parallel, each into its own table. + // Tested to be faster than walking halves. Using separate tables + // avoids the load/store dependencies that arise when two of the + // streams hit the same byte value, which is common for text. + var h0, h1, h2, h3 [256]uint16 + // Make size divisible by 4 for len(b)&3 != 0 { - h[b[0]]++ + h0[b[0]]++ b = b[1:] } n := len(b) / 4 x, y, z, w := b[:n], b[n:], b[n+n:], b[n+n+n:] y, z, w = y[:len(x)], z[:len(x)], w[:len(x)] for i, t := range x { - v0 := &h[t] - v1 := &h[y[i]] - v2 := &h[z[i]] - v3 := &h[w[i]] - *v0++ - *v1++ - *v2++ - *v3++ + h0[t]++ + h1[y[i]]++ + h2[z[i]]++ + h3[w[i]]++ + } + h = h[:256] + for i := range h { + h[i] += h0[i] + h1[i] + h2[i] + h3[i] } } --- a/vendor/github.com/klauspost/compress/flate/inflate.go +++ b/vendor/github.com/klauspost/compress/flate/inflate.go @@ -14,6 +14,8 @@ "io" "math/bits" "sync" + + "github.com/klauspost/compress/internal/regmask" ) const ( @@ -557,8 +559,8 @@ return err } } - rep += int(f.b & uint32(1<<(nb®SizeMaskUint32)-1)) - f.b >>= nb & regSizeMaskUint32 + rep += int(f.b & uint32(1<<(nb®mask.Shift32ByUint)-1)) + f.b >>= nb & regmask.Shift32ByUint f.nb -= nb if i+rep > n { if debugDecode { @@ -731,7 +733,7 @@ return noEOF(err) } f.roffset++ - f.b |= uint32(c) << (f.nb & regSizeMaskUint32) + f.b |= uint32(c) << (f.nb & regmask.Shift32ByUint) f.nb += 8 return nil } @@ -756,7 +758,7 @@ return 0, noEOF(err) } f.roffset++ - b |= uint32(c) << (nb & regSizeMaskUint32) + b |= uint32(c) << (nb & regmask.Shift32ByUint) nb += 8 } chunk := h.chunks[b&(huffmanNumChunks-1)] @@ -775,7 +777,7 @@ f.err = CorruptInputError(f.roffset) return 0, f.err } - f.b = b >> (n & regSizeMaskUint32) + f.b = b >> (n & regmask.Shift32ByUint) f.nb = nb - n return int(chunk >> huffmanValueShift), nil } --- a/vendor/github.com/klauspost/compress/flate/inflate_gen.go +++ b/vendor/github.com/klauspost/compress/flate/inflate_gen.go @@ -8,8 +8,19 @@ "fmt" "math/bits" "strings" + + "github.com/klauspost/compress/internal/regmask" ) +func peekBufio(fr *bufio.Reader) ([]byte, error) { + if fr.Buffered() == 0 { + if _, err := fr.Peek(1); err != nil && fr.Buffered() == 0 { + return nil, err + } + } + return fr.Peek(fr.Buffered()) +} + // Decode a single Huffman block from f. // hl and hd are the Huffman states for the lit/length values // and the distance values, respectively. If hd == nil, using the @@ -53,7 +64,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] @@ -71,7 +82,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n v = int(chunk >> huffmanValueShift) break @@ -109,15 +120,15 @@ if debugDecode { fmt.Println("morebits n>0:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 + fb >>= n & regmask.Shift32ByUint fnb -= n default: if debugDecode { @@ -137,11 +148,11 @@ if debugDecode { fmt.Println("morebits f.nb<5:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) @@ -165,7 +176,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] @@ -183,7 +194,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n dist = uint32(chunk >> huffmanValueShift) break @@ -197,7 +208,7 @@ case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) + extra := (dist & 1) << (nb & regmask.Shift32ByUint) for fnb < nb { c, err := fr.ReadByte() if err != nil { @@ -205,17 +216,17 @@ if debugDecode { fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fb >>= nb & regmask.Shift32ByUint fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + dist = 1<<((nb+1)®mask.Shift32ByUint) + 1 + extra // slower: dist = bitMask32[nb+1] + 2 + extra default: f.b, f.nb = fb, fnb @@ -304,7 +315,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] @@ -322,7 +333,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n v = int(chunk >> huffmanValueShift) break @@ -360,15 +371,15 @@ if debugDecode { fmt.Println("morebits n>0:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 + fb >>= n & regmask.Shift32ByUint fnb -= n default: if debugDecode { @@ -388,11 +399,11 @@ if debugDecode { fmt.Println("morebits f.nb<5:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) @@ -416,7 +427,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] @@ -434,7 +445,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n dist = uint32(chunk >> huffmanValueShift) break @@ -448,7 +459,7 @@ case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) + extra := (dist & 1) << (nb & regmask.Shift32ByUint) for fnb < nb { c, err := fr.ReadByte() if err != nil { @@ -456,17 +467,17 @@ if debugDecode { fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fb >>= nb & regmask.Shift32ByUint fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + dist = 1<<((nb+1)®mask.Shift32ByUint) + 1 + extra // slower: dist = bitMask32[nb+1] + 2 + extra default: f.b, f.nb = fb, fnb @@ -527,6 +538,8 @@ // but is smart enough to keep local variables in registers, so use nb and b, // inline call to moreBits and reassign b,nb back to f on return. fnb, fb, dict := f.nb, f.b, &f.dict + pbuf, _ := fr.Peek(fr.Buffered()) + pos := 0 switch f.stepState { case stateInit: @@ -548,14 +561,21 @@ n := uint(f.hl.maxRead) for { for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return + if pos >= len(pbuf) { + fr.Discard(pos) + var err error + pbuf, err = peekBufio(fr) + pos = 0 + if len(pbuf) == 0 { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } } + c := pbuf[pos] + pos++ f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] @@ -566,6 +586,7 @@ } if n <= fnb { if n == 0 { + fr.Discard(pos) f.b, f.nb = fb, fnb if debugDecode { fmt.Println("huffsym: n==0") @@ -573,7 +594,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n v = int(chunk >> huffmanValueShift) break @@ -586,6 +607,7 @@ case v < 256: dict.writeByte(byte(v)) if dict.availWrite() == 0 { + fr.Discard(pos) f.toRead = dict.readFlush() f.step = huffmanBufioReader f.stepState = stateInit @@ -594,6 +616,7 @@ } goto readLiteral case v == 256: + fr.Discard(pos) f.b, f.nb = fb, fnb f.finishBlock() return @@ -605,23 +628,31 @@ length = int(val.length) + 3 n := uint(val.extra) for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits n>0:", err) + if pos >= len(pbuf) { + fr.Discard(pos) + var err error + pbuf, err = peekBufio(fr) + pos = 0 + if len(pbuf) == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits n>0:", err) + } + f.err = noEOF(err) + return } - f.err = err - return } + c := pbuf[pos] + pos++ f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 + fb >>= n & regmask.Shift32ByUint fnb -= n default: + fr.Discard(pos) if debugDecode { fmt.Println(v, ">= maxNumLit") } @@ -633,17 +664,24 @@ var dist uint32 if f.hd == nil { for fnb < 5 { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb<5:", err) + if pos >= len(pbuf) { + fr.Discard(pos) + var err error + pbuf, err = peekBufio(fr) + pos = 0 + if len(pbuf) == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb<5:", err) + } + f.err = noEOF(err) + return } - f.err = err - return } + c := pbuf[pos] + pos++ f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) @@ -660,14 +698,21 @@ // inline call to moreBits and reassign b,nb back to f on return. for { for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return + if pos >= len(pbuf) { + fr.Discard(pos) + var err error + pbuf, err = peekBufio(fr) + pos = 0 + if len(pbuf) == 0 { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } } + c := pbuf[pos] + pos++ f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] @@ -678,6 +723,7 @@ } if n <= fnb { if n == 0 { + fr.Discard(pos) f.b, f.nb = fb, fnb if debugDecode { fmt.Println("huffsym: n==0") @@ -685,7 +731,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n dist = uint32(chunk >> huffmanValueShift) break @@ -699,27 +745,35 @@ case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) + extra := (dist & 1) << (nb & regmask.Shift32ByUint) for fnb < nb { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb= len(pbuf) { + fr.Discard(pos) + var err error + pbuf, err = peekBufio(fr) + pos = 0 + if len(pbuf) == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fb >>= nb & regmask.Shift32ByUint fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + dist = 1<<((nb+1)®mask.Shift32ByUint) + 1 + extra // slower: dist = bitMask32[nb+1] + 2 + extra default: + fr.Discard(pos) f.b, f.nb = fb, fnb if debugDecode { fmt.Println("dist too big:", dist, maxNumDist) @@ -730,6 +784,7 @@ // No check on length; encoding can be prescient. if dist > uint32(dict.histSize()) { + fr.Discard(pos) f.b, f.nb = fb, fnb if debugDecode { fmt.Println("dist > dict.histSize():", dist, dict.histSize()) @@ -752,6 +807,7 @@ f.copyLen -= cnt if dict.availWrite() == 0 || f.copyLen > 0 { + fr.Discard(pos) f.toRead = dict.readFlush() f.step = huffmanBufioReader // We need to continue this work f.stepState = stateDict @@ -806,7 +862,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] @@ -824,7 +880,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n v = int(chunk >> huffmanValueShift) break @@ -862,15 +918,15 @@ if debugDecode { fmt.Println("morebits n>0:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 + fb >>= n & regmask.Shift32ByUint fnb -= n default: if debugDecode { @@ -890,11 +946,11 @@ if debugDecode { fmt.Println("morebits f.nb<5:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) @@ -918,7 +974,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] @@ -936,7 +992,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n dist = uint32(chunk >> huffmanValueShift) break @@ -950,7 +1006,7 @@ case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) + extra := (dist & 1) << (nb & regmask.Shift32ByUint) for fnb < nb { c, err := fr.ReadByte() if err != nil { @@ -958,17 +1014,17 @@ if debugDecode { fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fb >>= nb & regmask.Shift32ByUint fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + dist = 1<<((nb+1)®mask.Shift32ByUint) + 1 + extra // slower: dist = bitMask32[nb+1] + 2 + extra default: f.b, f.nb = fb, fnb @@ -1057,7 +1113,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] @@ -1075,7 +1131,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n v = int(chunk >> huffmanValueShift) break @@ -1113,15 +1169,15 @@ if debugDecode { fmt.Println("morebits n>0:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 + fb >>= n & regmask.Shift32ByUint fnb -= n default: if debugDecode { @@ -1141,11 +1197,11 @@ if debugDecode { fmt.Println("morebits f.nb<5:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) @@ -1169,7 +1225,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] @@ -1187,7 +1243,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n dist = uint32(chunk >> huffmanValueShift) break @@ -1201,7 +1257,7 @@ case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) + extra := (dist & 1) << (nb & regmask.Shift32ByUint) for fnb < nb { c, err := fr.ReadByte() if err != nil { @@ -1209,17 +1265,17 @@ if debugDecode { fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fb >>= nb & regmask.Shift32ByUint fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + dist = 1<<((nb+1)®mask.Shift32ByUint) + 1 + extra // slower: dist = bitMask32[nb+1] + 2 + extra default: f.b, f.nb = fb, fnb --- a/vendor/github.com/klauspost/compress/flate/matchlen_generic.go +++ b/vendor/github.com/klauspost/compress/flate/matchlen_generic.go @@ -12,18 +12,17 @@ // matchLen returns the maximum common prefix length of a and b. // a must be the shortest of the two. func matchLen(a, b []byte) (n int) { - left := len(a) - for left >= 8 { - diff := le.Load64(a, n) ^ le.Load64(b, n) + // Keeping a and b the same length and reslicing them as we go + // lets the compiler prove that no bounds checks are needed. + b = b[:len(a)] + for len(a) >= 8 { + diff := le.Load64(a, 0) ^ le.Load64(b, 0) if diff != 0 { return n + bits.TrailingZeros64(diff)>>3 } + a, b = a[8:], b[8:] n += 8 - left -= 8 } - - a = a[n:] - b = b[n:] for i := range a { if a[i] != b[i] { break --- a/vendor/github.com/klauspost/compress/flate/token.go +++ b/vendor/github.com/klauspost/compress/flate/token.go @@ -215,7 +215,11 @@ ux &= -0x7f800001 ux += 127 << 23 uval := math.Float32frombits(uint32(ux)) - log2 += ((-0.34484843)*uval+2.02466578)*uval - 0.67487759 + + p := float32(-0.34484843*uval) + 2.02466578 + q := float32(p*uval) - 0.67487759 + log2 += q + return log2 } @@ -232,7 +236,8 @@ for _, v := range t.litHist[:] { if v > 0 { n := float32(v) - shannon += atLeastOne(-mFastLog2(n*invTotal)) * n + w := float32(atLeastOne(-mFastLog2(n*invTotal)) * n) + shannon += w } } // Just add 15 for EOB @@ -240,7 +245,8 @@ for i, v := range t.extraHist[1 : literalCount-256] { if v > 0 { n := float32(v) - shannon += atLeastOne(-mFastLog2(n*invTotal)) * n + w := float32(atLeastOne(-mFastLog2(n*invTotal)) * n) + shannon += w bits += int(lengthExtraBits[i&31]) * int(v) nMatches += int(v) } @@ -251,7 +257,8 @@ for i, v := range t.offHist[:offsetCodeCount] { if v > 0 { n := float32(v) - shannon += atLeastOne(-mFastLog2(n*invTotal)) * n + w := float32(atLeastOne(-mFastLog2(n*invTotal)) * n) + shannon += w bits += int(offsetExtraBits[i&31]) * int(v) } } @@ -358,9 +365,6 @@ func (t token) length() uint8 { return uint8(t >> lengthShift) } -// Convert length to code. -func lengthCode(len uint8) uint8 { return lengthCodes[len] } - // Returns the offset code corresponding to a specific offset func offsetCode(off uint32) uint32 { if false { --- a/vendor/github.com/klauspost/compress/fse/compress.go +++ b/vendor/github.com/klauspost/compress/fse/compress.go @@ -659,7 +659,8 @@ if v >= 0 { total += int(v) } else { - total -= int(v) + // -1 takes one cell. Adding -v would let a wrapped -32768 pass. + total++ } } defer func() { --- a/vendor/github.com/klauspost/compress/fse/decompress.go +++ b/vendor/github.com/klauspost/compress/fse/decompress.go @@ -3,6 +3,7 @@ import ( "errors" "fmt" + "math" ) const ( @@ -123,6 +124,11 @@ remaining -= count gotTotal += count } + if count > math.MaxInt16 { + // Only one symbol filling a tableLog 15 table, which cannot be decoded. + // Truncated, it would leave the decoding table unwritten. + return fmt.Errorf("corruption detected (count %d > %d)", count, math.MaxInt16) + } s.norm[charnum&0xff] = int16(count) charnum++ previous0 = count == 0 --- a/vendor/github.com/klauspost/compress/huff0/bitreader.go +++ b/vendor/github.com/klauspost/compress/huff0/bitreader.go @@ -9,6 +9,7 @@ "errors" "fmt" "io" + "math/bits" "github.com/klauspost/compress/internal/le" ) @@ -210,6 +211,68 @@ return b.off*8 + uint(64-b.bitsRead) } +// canUseAsm reports whether the reader has a full 8-byte window ahead of +// its read pointer, which the Decompress4X asm loops need to take over. +func (b *bitReaderShifted) canUseAsm() bool { + return b.off >= 8 +} + +// prepareForAsm establishes the invariant the Decompress4X asm loops rely +// on: value == load64(in[off:off+8]) << bitsRead with bitsRead <= 7, so +// that a full group of symbols can never shift their sentinel bit out of +// the container. init leaves bitsRead == 8 when the final byte of the +// stream is exactly 0x01; that whole byte is consumed, so the same position +// is the window one byte lower with nothing consumed. Requires canUseAsm. +func (b *bitReaderShifted) prepareForAsm() { + if b.bitsRead >= 8 { + b.off-- + b.value = le.Load64(b.in, b.off) + b.bitsRead -= 8 + } +} + +// restoreFromAsm converts the state left behind by the Decompress4X asm +// loops back to the invariant the Go code relies on: value holds the 8 +// bytes at in[off:off+8] shifted left by bitsRead, with the low bitsRead +// bits zero. +// +// The asm keeps a sentinel bit in value just below the unread bits, so its +// trailing zero count is the number of consumed bits. The sentinel is ORed +// over the lowest bit of the window, which the loop never consumes before +// re-reading memory, so the window is re-read here too rather than taken +// from value. The asm reports off as the signed distance from the start of +// the stream, and it can be negative: a reload always reads a whole 8-byte +// window, so a stream that is nearly drained ends with up to 7 bytes of the +// previous stream (or the jump table) below its start inside the window. +// Those bytes sit below the stream's own bits and count as consumed. +// Anything further below, or more bits consumed than the stream holds, is +// corruption. +func (b *bitReaderShifted) restoreFromAsm() error { + off := int(b.off) + consumed := uint(bits.TrailingZeros64(b.value)) + if off < 0 { + if off < -7 { + return errors.New("corruption detected: stream underrun") + } + consumed += uint(-off) * 8 + if consumed > 64 { + return errors.New("corruption detected: stream underrun") + } + off = 0 + } + if off+8 > len(b.in) { + return errors.New("corruption detected: stream overrun") + } + b.off = uint(off) + b.bitsRead = uint8(consumed) + if consumed >= 64 { + b.value = 0 + } else { + b.value = le.Load64(b.in, b.off) << consumed + } + return nil +} + // close the bitstream and returns an error if out-of-buffer reads occurred. func (b *bitReaderShifted) close() error { // Release reference. --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -54,6 +54,10 @@ if len(in) < int(iSize) { return s, nil, fmt.Errorf("input too small for table, want %d bytes, have %d", iSize, len(in)) } + // The low nibble is the weights' FSE tableLog - 5, which the format limits to 6. + if in[0]&15+5 > 6 { + return s, nil, errors.New("corrupt input: weights tableLog too big") + } // FSE compressed weights s.fse.DecompressLimit = 255 hw := s.huffWeight[:] --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -1,222 +1,128 @@ //go:build amd64 && !appengine && !noasm && gc -// This file contains the specialisation of Decoder.Decompress4X -// and Decoder.Decompress1X that use an asm implementation of thir main loops. +// amd64 stubs and dispatch for the asm loops used by decompress_asm.go. package huff0 import ( - "errors" - "fmt" - "github.com/klauspost/compress/internal/cpuinfo" ) -// decompress4x_main_loop_x86 is an x86 assembler implementation -// of Decompress4X when tablelog > 8. +// decompress4x_main_loop_amd64 is an x86 assembler implementation +// of Decompress4X when tablelog > 8, decoding fastSymbols symbols per +// stream between bit container reloads. // //go:noescape func decompress4x_main_loop_amd64(ctx *decompress4xContext) -// decompress4x_8b_loop_x86 is an x86 assembler implementation -// of Decompress4X when tablelog <= 8 which decodes 4 entries -// per loop. +// decompress4x_8b_main_loop_amd64 is an x86 assembler implementation +// of Decompress4X when tablelog <= 8, decoding fast8bSymbols symbols +// per stream between bit container reloads. // //go:noescape func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) -// fallback8BitSize is the size where using Go version is faster. -const fallback8BitSize = 800 +// decompress4x_4b_main_loop_amd64 is an x86 assembler implementation +// of Decompress4X when tablelog <= 4, decoding fast4bSymbols symbols +// per stream between bit container reloads. +// +//go:noescape +func decompress4x_4b_main_loop_amd64(ctx *decompress4xContext) -type decompress4xContext struct { - pbr *[4]bitReaderShifted - peekBits uint8 - out *byte - dstEvery int - tbl *dEntrySingle - decoded int - limit *byte -} - -// Decompress4X will decompress a 4X encoded stream. -// The length of the supplied input must match the end of a block exactly. -// The *capacity* of the dst slice must match the destination size of -// the uncompressed data exactly. -func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - if len(src) < 6+(4*1) { - return nil, errors.New("input too small") - } - - use8BitTables := d.actualTableLog <= 8 - if cap(dst) < fallback8BitSize && use8BitTables { - return d.decompress4X8bit(dst, src) - } - - var br [4]bitReaderShifted - // Decode "jump table" - start := 6 - for i := range 3 { - length := int(src[i*2]) | (int(src[i*2+1]) << 8) - if start+length >= len(src) { - return nil, errors.New("truncated input (or invalid offset)") - } - err := br[i].init(src[start : start+length]) - if err != nil { - return nil, err - } - start += length - } - err := br[3].init(src[start:]) - if err != nil { - return nil, err - } - - // destination, offset to match first output - dstSize := cap(dst) - dst = dst[:dstSize] - out := dst - dstEvery := (dstSize + 3) / 4 - - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - single := d.dt.single[:tlSize] - - var decoded int - - if len(out) > 4*4 && !(br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4) { - ctx := decompress4xContext{ - pbr: &br, - peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() - out: &out[0], - dstEvery: dstEvery, - tbl: &single[0], - limit: &out[dstEvery-4], // Always stop decoding when first buffer gets here to avoid writing OOB on last. - } - if use8BitTables { - decompress4x_8b_main_loop_amd64(&ctx) - } else { - decompress4x_main_loop_amd64(&ctx) - } - - decoded = ctx.decoded - out = out[decoded/4:] - } - - // Decode remaining. - remainBytes := dstEvery - (decoded / 4) - for i := range br { - offset := dstEvery * i - endsAt := min(offset+remainBytes, len(out)) - br := &br[i] - bitsLeft := br.remaining() - for bitsLeft > 0 { - br.fill() - if offset >= endsAt { - return nil, errors.New("corruption detected: stream overrun 4") - } - - // Read value and increment offset. - val := br.peekBitsFast(d.actualTableLog) - v := single[val&tlMask].entry - nBits := uint8(v) - br.advance(nBits) - bitsLeft -= uint(nBits) - out[offset] = uint8(v >> 8) - offset++ - } - if offset != endsAt { - return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) - } - decoded += offset - dstEvery*i - err = br.close() - if err != nil { - return nil, err - } - } - if dstSize != decoded { - return nil, errors.New("corruption detected: short output block") - } - return dst, nil -} +// decompress4x_4b_main_loop_bmi2 is the BMI2 twin of decompress4x_4b_main_loop_amd64. +// +//go:noescape +func decompress4x_4b_main_loop_bmi2(ctx *decompress4xContext) + +// decompress4x_main_loop_bmi2 is the BMI2 twin of decompress4x_main_loop_amd64. +// +//go:noescape +func decompress4x_main_loop_bmi2(ctx *decompress4xContext) -// decompress4x_main_loop_x86 is an x86 assembler implementation -// of Decompress1X when tablelog > 8. +// decompress4x_8b_main_loop_bmi2 is the BMI2 twin of decompress4x_8b_main_loop_amd64. +// +//go:noescape +func decompress4x_8b_main_loop_bmi2(ctx *decompress4xContext) + +// decompress1x_main_loop_amd64 is an x86 assembler implementation +// of Decompress1X when tablelog > 8, decoding fastSymbols symbols +// between bit container reloads. // //go:noescape func decompress1x_main_loop_amd64(ctx *decompress1xContext) -// decompress4x_main_loop_x86 is an x86 with BMI2 assembler implementation -// of Decompress1X when tablelog > 8. +// decompress1x_8b_main_loop_amd64 is an x86 assembler implementation +// of Decompress1X when tablelog <= 8, decoding fast8bSymbols symbols +// between bit container reloads. +// +//go:noescape +func decompress1x_8b_main_loop_amd64(ctx *decompress1xContext) + +// decompress1x_4b_main_loop_amd64 is an x86 assembler implementation +// of Decompress1X when tablelog <= 4, decoding fast4bSymbols symbols +// between bit container reloads. +// +//go:noescape +func decompress1x_4b_main_loop_amd64(ctx *decompress1xContext) + +// decompress1x_main_loop_bmi2 is the BMI2 twin of decompress1x_main_loop_amd64. // //go:noescape func decompress1x_main_loop_bmi2(ctx *decompress1xContext) -type decompress1xContext struct { - pbr *bitReaderShifted - peekBits uint8 - out *byte - outCap int - tbl *dEntrySingle - decoded int -} - -// Error reported by asm implementations -const error_max_decoded_size_exeeded = -1 - -// Decompress1X will decompress a 1X encoded stream. -// The cap of the output buffer will be the maximum decompressed size. -// The length of the supplied input must match the end of a block exactly. -func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - var br bitReaderShifted - err := br.init(src) - if err != nil { - return dst, err - } - maxDecodedSize := cap(dst) - dst = dst[:maxDecodedSize] - - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - - if maxDecodedSize >= 4 { - ctx := decompress1xContext{ - pbr: &br, - out: &dst[0], - outCap: maxDecodedSize, - peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() - tbl: &d.dt.single[0], - } - - if cpuinfo.HasBMI2() { - decompress1x_main_loop_bmi2(&ctx) - } else { - decompress1x_main_loop_amd64(&ctx) - } - if ctx.decoded == error_max_decoded_size_exeeded { - return nil, ErrMaxDecodedSizeExceeded - } - - dst = dst[:ctx.decoded] - } - - // br < 8, so uint8 is fine - bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead - for bitsLeft > 0 { - br.fill() - if len(dst) >= maxDecodedSize { - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] - nBits := uint8(v.entry) - br.advance(nBits) - bitsLeft -= nBits - dst = append(dst, uint8(v.entry>>8)) +// decompress1x_8b_main_loop_bmi2 is the BMI2 twin of decompress1x_8b_main_loop_amd64. +// +//go:noescape +func decompress1x_8b_main_loop_bmi2(ctx *decompress1xContext) + +// decompress1x_4b_main_loop_bmi2 is the BMI2 twin of decompress1x_4b_main_loop_amd64. +// +//go:noescape +func decompress1x_4b_main_loop_bmi2(ctx *decompress1xContext) + +func decompress4x_main_loop_asm(ctx *decompress4xContext) { + if cpuinfo.HasBMI2() { + decompress4x_main_loop_bmi2(ctx) + } else { + decompress4x_main_loop_amd64(ctx) + } +} + +func decompress4x_8b_main_loop_asm(ctx *decompress4xContext) { + if cpuinfo.HasBMI2() { + decompress4x_8b_main_loop_bmi2(ctx) + } else { + decompress4x_8b_main_loop_amd64(ctx) + } +} + +func decompress4x_4b_main_loop_asm(ctx *decompress4xContext) { + if cpuinfo.HasBMI2() { + decompress4x_4b_main_loop_bmi2(ctx) + } else { + decompress4x_4b_main_loop_amd64(ctx) + } +} + +func decompress1x_main_loop_asm(ctx *decompress1xContext) { + if cpuinfo.HasBMI2() { + decompress1x_main_loop_bmi2(ctx) + } else { + decompress1x_main_loop_amd64(ctx) + } +} + +func decompress1x_8b_main_loop_asm(ctx *decompress1xContext) { + if cpuinfo.HasBMI2() { + decompress1x_8b_main_loop_bmi2(ctx) + } else { + decompress1x_8b_main_loop_amd64(ctx) + } +} + +func decompress1x_4b_main_loop_asm(ctx *decompress1xContext) { + if cpuinfo.HasBMI2() { + decompress1x_4b_main_loop_bmi2(ctx) + } else { + decompress1x_4b_main_loop_amd64(ctx) } - return dst, br.close() } --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s @@ -1,830 +1,3386 @@ -// Code generated by command: go run gen.go -out ../decompress_amd64.s -pkg=huff0. DO NOT EDIT. +// Code generated by command: go run gen.go -out ../decompress.s -arch amd64,arm64 -pkg=huff0. DO NOT EDIT. -//go:build amd64 && !appengine && !noasm && gc +//go:build !appengine && !noasm && gc // func decompress4x_main_loop_amd64(ctx *decompress4xContext) +// Requires: BMI, CMOV TEXT ·decompress4x_main_loop_amd64(SB), $0-8 - // Preload values - MOVQ ctx+0(FP), AX - MOVBQZX 8(AX), DI - MOVQ 16(AX), BX - MOVQ 48(AX), SI - MOVQ 24(AX), R8 - MOVQ 32(AX), R9 - MOVQ (AX), R10 - - // Main loop -main_loop: - XORL DX, DX - CMPQ BX, SI - SETGE DL - - // br0.fillFast32() - MOVQ 32(R10), R11 - MOVBQZX 40(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill0 - MOVQ 24(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ (R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 24(R10) - ORQ R13, R11 - - // exhausted += (br0.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill0: - // val0 := br0.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br0.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br0.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX - - // br0.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX) - - // update the bitreader structure - MOVQ R11, 32(R10) - MOVB R12, 40(R10) - - // br1.fillFast32() - MOVQ 80(R10), R11 - MOVBQZX 88(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill1 - MOVQ 72(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 48(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 72(R10) - ORQ R13, R11 - - // exhausted += (br1.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill1: - // val0 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br1.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br1.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX - - // br1.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX)(R8*1) - - // update the bitreader structure - MOVQ R11, 80(R10) - MOVB R12, 88(R10) - - // br2.fillFast32() - MOVQ 128(R10), R11 - MOVBQZX 136(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill2 - MOVQ 120(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 96(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 120(R10) - ORQ R13, R11 - - // exhausted += (br2.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill2: - // val0 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br2.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX)(R8*2) - - // update the bitreader structure - MOVQ R11, 128(R10) - MOVB R12, 136(R10) - - // br3.fillFast32() - MOVQ 176(R10), R11 - MOVBQZX 184(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill3 - MOVQ 168(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 144(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 168(R10) - ORQ R13, R11 - - // exhausted += (br3.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill3: - // val0 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br3.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br3.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX + MOVQ ctx+0(FP), AX - // br3.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - LEAQ (R8)(R8*2), CX - MOVW AX, (BX)(CX*1) - - // update the bitreader structure - MOVQ R11, 176(R10) - MOVB R12, 184(R10) - ADDQ $0x02, BX - TESTB DL, DL - JZ main_loop - MOVQ ctx+0(FP), AX - SUBQ 16(AX), BX - SHLQ $0x02, BX - MOVQ BX, 40(AX) + // Preload values + MOVBQZX 8(AX), DX + MOVQ 32(AX), BX + MOVQ 16(AX), SI + MOVQ 24(AX), CX + LEAQ (SI)(CX*1), R8 + LEAQ (SI)(CX*2), R10 + LEAQ (CX)(CX*2), R12 + ADDQ SI, R12 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX) - - // update the bitreader structure - MOVQ R11, 32(R10) - MOVB R12, 40(R10) - - // br1.fillFast32() - MOVQ 80(R10), R11 - MOVBQZX 88(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill1 - MOVQ 72(R10), R13 - SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 48(R10), R14 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 - MOVQ R12, CX - SHLQ CL, R14 - MOVQ R13, 72(R10) - ORQ R14, R11 - - // exhausted += (br1.off < 4) - CMPQ R13, $0x04 - ADCB $+0, DL + TZCNTQ R13, R14 + MOVQ R14, CX + ANDQ $0x07, CX + SHRQ $0x03, R14 + MOVQ 88(AX), R13 + SUBQ R14, R13 + MOVQ R13, 88(AX) + MOVQ (R13), R13 + ORQ $0x01, R13 + SHLQ CL, R13 + ADDQ $0x0e, SI + ADDQ $0x0e, R8 + ADDQ $0x0e, R10 + ADDQ $0x0e, R12 + CMPQ SI, 96(AX) + JB inner_loop + JMP outer_loop + +done: + // Hand the state back: off = ip - in (negative when the window reached below the stream start), + // value = the sentinel-form container. bitReaderShifted.restoreFromAsm normalizes both. + MOVQ (AX), CX + MOVQ 64(AX), DX + SUBQ (CX), DX + MOVQ DX, 24(CX) + MOVQ DI, 32(CX) + MOVQ 72(AX), DX + SUBQ 48(CX), DX + MOVQ DX, 72(CX) + MOVQ R9, 80(CX) + MOVQ 80(AX), DX + SUBQ 96(CX), DX + MOVQ DX, 120(CX) + MOVQ R11, 128(CX) + MOVQ 88(AX), DX + SUBQ 144(CX), DX + MOVQ DX, 168(CX) + MOVQ R13, 176(CX) + SUBQ 16(AX), SI + SHLQ $0x02, SI + MOVQ SI, 40(AX) + RET -skip_fill1: - // val0 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 +// func decompress4x_main_loop_bmi2(ctx *decompress4xContext) +// Requires: BMI, BMI2, CMOV +TEXT ·decompress4x_main_loop_bmi2(SB), $0-8 + MOVQ ctx+0(FP), AX - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + LEAQ (BX)(SI*1), DI + LEAQ (BX)(SI*2), R9 + LEAQ (SI)(SI*2), R11 + ADDQ BX, R11 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX)(R8*1) - - // update the bitreader structure - MOVQ R11, 80(R10) - MOVB R12, 88(R10) - - // br2.fillFast32() - MOVQ 128(R10), R11 - MOVBQZX 136(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill2 - MOVQ 120(R10), R13 - SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 96(R10), R14 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 - MOVQ R12, CX + // Refill the whole bytes consumed from below the window + MOVQ R11, R13 + ANDQ $0x38, R13 + MOVQ -8(R8), R14 + SHRQ $0x01, R14 + ANDQ $0x07, R11 + MOVQ R13, CX + XORQ $0x3f, CX + SHRQ CL, R14 + MOVQ R11, CX SHLQ CL, R14 - MOVQ R13, 120(R10) - ORQ R14, R11 + ORQ R14, R10 + SHRQ $0x03, R13 + SUBQ R13, R8 + ADDQ $0x05, SI + CMPQ SI, R12 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), R8 + MOVQ R8, 24(CX) + MOVQ R10, 32(CX) + MOVB R11, 40(CX) + SUBQ 16(AX), SI + MOVQ SI, 40(AX) + RET - // exhausted += (br2.off < 4) - CMPQ R13, $0x04 - ADCB $+0, DL +// func decompress1x_8b_main_loop_amd64(ctx *decompress1xContext) +// Requires: CMOV +TEXT ·decompress1x_8b_main_loop_amd64(SB), $0-8 + MOVQ ctx+0(FP), AX -skip_fill2: - // val0 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // Preload values + MOVBQZX 8(AX), DX + MOVQ 32(AX), BX + MOVQ 16(AX), SI + MOVQ 24(AX), DI + ADDQ SI, DI + MOVQ 56(AX), R8 + MOVQ 48(AX), R9 + MOVQ (AX), CX + MOVQ 32(CX), R10 + MOVBQZX 40(CX), R11 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 + MOVQ DI, CX + SUBQ SI, CX + JLE done + SHRQ $0x03, CX + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ R8, R12 + SUBQ R9, R12 + SHRQ $0x03, R12 + CMPQ R12, CX + CMOVQCS R12, CX + TESTQ CX, CX + JZ done + IMUL3Q $0x07, CX, R12 + ADDQ SI, R12 + +inner_loop: + // symbol 0 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, (SI) - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX + // symbol 1 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 1(SI) - // br2.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 + // symbol 2 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 2(SI) - // val1 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // symbol 3 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 3(SI) - // v1 := table[val0&mask] - MOVW (R9)(R13*2), CX + // symbol 4 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 4(SI) - // br2.advance(uint8(v1.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX + // symbol 5 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 5(SI) - // val2 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // symbol 6 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 6(SI) - // v2 := table[val0&mask] - MOVW (R9)(R13*2), CX + // Refill the whole bytes consumed from below the window + MOVQ R11, R13 + ANDQ $0x38, R13 + MOVQ -8(R8), R14 + SHRQ $0x01, R14 + ANDQ $0x07, R11 + MOVQ R13, CX + XORQ $0x3f, CX + SHRQ CL, R14 + MOVQ R11, CX + SHLQ CL, R14 + ORQ R14, R10 + SHRQ $0x03, R13 + SUBQ R13, R8 + ADDQ $0x07, SI + CMPQ SI, R12 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), R8 + MOVQ R8, 24(CX) + MOVQ R10, 32(CX) + MOVB R11, 40(CX) + SUBQ 16(AX), SI + MOVQ SI, 40(AX) + RET - // br2.advance(uint8(v2.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 +// func decompress1x_4b_main_loop_amd64(ctx *decompress1xContext) +// Requires: CMOV +TEXT ·decompress1x_4b_main_loop_amd64(SB), $0-8 + MOVQ ctx+0(FP), AX - // val3 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // Preload values + MOVBQZX 8(AX), DX + MOVQ 32(AX), BX + MOVQ 16(AX), SI + MOVQ 24(AX), DI + ADDQ SI, DI + MOVQ 56(AX), R8 + MOVQ 48(AX), R9 + MOVQ (AX), CX + MOVQ 32(CX), R10 + MOVBQZX 40(CX), R11 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 16 + MOVQ DI, CX + SUBQ SI, CX + JLE done + SHRQ $0x04, CX + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ R8, R12 + SUBQ R9, R12 + SHRQ $0x03, R12 + CMPQ R12, CX + CMOVQCS R12, CX + TESTQ CX, CX + JZ done + IMUL3Q $0x0e, CX, R12 + ADDQ SI, R12 + +inner_loop: + // symbol 0 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, (SI) - // v3 := table[val0&mask] - MOVW (R9)(R13*2), CX + // symbol 1 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 1(SI) - // br2.advance(uint8(v3.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX + // symbol 2 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 2(SI) - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX)(R8*2) - - // update the bitreader structure - MOVQ R11, 128(R10) - MOVB R12, 136(R10) - - // br3.fillFast32() - MOVQ 176(R10), R11 - MOVBQZX 184(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill3 - MOVQ 168(R10), R13 - SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 144(R10), R14 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 - MOVQ R12, CX - SHLQ CL, R14 - MOVQ R13, 168(R10) - ORQ R14, R11 + // symbol 3 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 3(SI) - // exhausted += (br3.off < 4) - CMPQ R13, $0x04 - ADCB $+0, DL + // symbol 4 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 4(SI) -skip_fill3: - // val0 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // symbol 5 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 5(SI) - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX + // symbol 6 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 6(SI) - // br3.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 + // symbol 7 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 7(SI) - // val1 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // symbol 8 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 8(SI) - // v1 := table[val0&mask] - MOVW (R9)(R13*2), CX + // symbol 9 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 9(SI) - // br3.advance(uint8(v1.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX + // symbol 10 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 10(SI) - // val2 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // symbol 11 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 11(SI) - // v2 := table[val0&mask] - MOVW (R9)(R13*2), CX + // symbol 12 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 12(SI) - // br3.advance(uint8(v2.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 + // symbol 13 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 13(SI) - // val3 := br3.peekTopBits(peekBits) + // Refill the whole bytes consumed from below the window MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + ANDQ $0x38, R13 + MOVQ -8(R8), R14 + SHRQ $0x01, R14 + ANDQ $0x07, R11 + MOVQ R13, CX + XORQ $0x3f, CX + SHRQ CL, R14 + MOVQ R11, CX + SHLQ CL, R14 + ORQ R14, R10 + SHRQ $0x03, R13 + SUBQ R13, R8 + ADDQ $0x0e, SI + CMPQ SI, R12 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), R8 + MOVQ R8, 24(CX) + MOVQ R10, 32(CX) + MOVB R11, 40(CX) + SUBQ 16(AX), SI + MOVQ SI, 40(AX) + RET - // v3 := table[val0&mask] - MOVW (R9)(R13*2), CX +// func decompress1x_main_loop_bmi2(ctx *decompress1xContext) +// Requires: BMI2, CMOV +TEXT ·decompress1x_main_loop_bmi2(SB), $0-8 + MOVQ ctx+0(FP), AX - // br3.advance(uint8(v3.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + ADDQ BX, SI + MOVQ 56(AX), DI + MOVQ 48(AX), R8 + MOVQ (AX), R10 + MOVQ 32(R10), R9 + MOVBQZX 40(R10), R10 - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - LEAQ (R8)(R8*2), CX - MOVL AX, (BX)(CX*1) - - // update the bitreader structure - MOVQ R11, 176(R10) - MOVB R12, 184(R10) - ADDQ $0x04, BX - TESTB DL, DL - JZ main_loop - MOVQ ctx+0(FP), AX - SUBQ 16(AX), BX - SHLQ $0x02, BX - MOVQ BX, 40(AX) +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 + MOVQ SI, R11 + SUBQ BX, R11 + JLE done + SHRQ $0x03, R11 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ DI, R12 + SUBQ R8, R12 + SHRQ $0x03, R12 + CMPQ R12, R11 + CMOVQCS R12, R11 + TESTQ R11, R11 + JZ done + IMUL3Q $0x05, R11, R11 + ADDQ BX, R11 + +inner_loop: + // symbol 0 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, (BX) + + // symbol 1 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 1(BX) + + // symbol 2 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 2(BX) + + // symbol 3 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 3(BX) + + // symbol 4 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 4(BX) + + // Refill the whole bytes consumed from below the window + MOVQ R10, R12 + ANDQ $0x38, R12 + MOVQ -8(DI), R13 + SHRQ $0x01, R13 + ANDQ $0x07, R10 + MOVQ R12, R14 + XORQ $0x3f, R14 + SHRXQ R14, R13, R13 + SHLXQ R10, R13, R13 + ORQ R13, R9 + SHRQ $0x03, R12 + SUBQ R12, DI + ADDQ $0x05, BX + CMPQ BX, R11 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), DI + MOVQ DI, 24(CX) + MOVQ R9, 32(CX) + MOVB R10, 40(CX) + SUBQ 16(AX), BX + MOVQ BX, 40(AX) RET -// func decompress1x_main_loop_amd64(ctx *decompress1xContext) -TEXT ·decompress1x_main_loop_amd64(SB), $0-8 - MOVQ ctx+0(FP), CX - MOVQ 16(CX), DX - MOVQ 24(CX), BX - CMPQ BX, $0x04 - JB error_max_decoded_size_exceeded - LEAQ (DX)(BX*1), BX - MOVQ (CX), SI - MOVQ (SI), R8 - MOVQ 24(SI), R9 - MOVQ 32(SI), R10 - MOVBQZX 40(SI), R11 - MOVQ 32(CX), SI - MOVBQZX 8(CX), DI - JMP loop_condition - -main_loop: - // Check if we have room for 4 bytes in the output buffer - LEAQ 4(DX), CX - CMPQ CX, BX - JGE error_max_decoded_size_exceeded - - // Decode 4 values - CMPQ R11, $0x20 - JL bitReader_fillFast_1_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), R12 - MOVQ R11, CX - SHLQ CL, R12 - ORQ R12, R10 +// func decompress1x_8b_main_loop_bmi2(ctx *decompress1xContext) +// Requires: BMI2, CMOV +TEXT ·decompress1x_8b_main_loop_bmi2(SB), $0-8 + MOVQ ctx+0(FP), AX -bitReader_fillFast_1_end: - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AL - MOVBQZX CL, CX - ADDQ CX, R11 - SHLQ CL, R10 - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AH - MOVBQZX CL, CX - ADDQ CX, R11 - SHLQ CL, R10 - BSWAPL AX - CMPQ R11, $0x20 - JL bitReader_fillFast_2_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), R12 - MOVQ R11, CX - SHLQ CL, R12 - ORQ R12, R10 - -bitReader_fillFast_2_end: - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AH - MOVBQZX CL, CX - ADDQ CX, R11 - SHLQ CL, R10 - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AL - MOVBQZX CL, CX - ADDQ CX, R11 - SHLQ CL, R10 - BSWAPL AX - - // Store the decoded values - MOVL AX, (DX) - ADDQ $0x04, DX - -loop_condition: - CMPQ R9, $0x08 - JGE main_loop - - // Update ctx structure - MOVQ ctx+0(FP), AX - SUBQ 16(AX), DX - MOVQ DX, 40(AX) - MOVQ (AX), AX - MOVQ R9, 24(AX) - MOVQ R10, 32(AX) - MOVB R11, 40(AX) + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + ADDQ BX, SI + MOVQ 56(AX), DI + MOVQ 48(AX), R8 + MOVQ (AX), R10 + MOVQ 32(R10), R9 + MOVBQZX 40(R10), R10 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 + MOVQ SI, R11 + SUBQ BX, R11 + JLE done + SHRQ $0x03, R11 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ DI, R12 + SUBQ R8, R12 + SHRQ $0x03, R12 + CMPQ R12, R11 + CMOVQCS R12, R11 + TESTQ R11, R11 + JZ done + IMUL3Q $0x07, R11, R11 + ADDQ BX, R11 + +inner_loop: + // symbol 0 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, (BX) + + // symbol 1 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 1(BX) + + // symbol 2 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 2(BX) + + // symbol 3 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 3(BX) + + // symbol 4 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 4(BX) + + // symbol 5 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 5(BX) + + // symbol 6 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 6(BX) + + // Refill the whole bytes consumed from below the window + MOVQ R10, R12 + ANDQ $0x38, R12 + MOVQ -8(DI), R13 + SHRQ $0x01, R13 + ANDQ $0x07, R10 + MOVQ R12, R14 + XORQ $0x3f, R14 + SHRXQ R14, R13, R13 + SHLXQ R10, R13, R13 + ORQ R13, R9 + SHRQ $0x03, R12 + SUBQ R12, DI + ADDQ $0x07, BX + CMPQ BX, R11 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), DI + MOVQ DI, 24(CX) + MOVQ R9, 32(CX) + MOVB R10, 40(CX) + SUBQ 16(AX), BX + MOVQ BX, 40(AX) RET - // Report error -error_max_decoded_size_exceeded: +// func decompress1x_4b_main_loop_bmi2(ctx *decompress1xContext) +// Requires: BMI2, CMOV +TEXT ·decompress1x_4b_main_loop_bmi2(SB), $0-8 MOVQ ctx+0(FP), AX - MOVQ $-1, CX - MOVQ CX, 40(AX) - RET -// func decompress1x_main_loop_bmi2(ctx *decompress1xContext) -// Requires: BMI2 -TEXT ·decompress1x_main_loop_bmi2(SB), $0-8 - MOVQ ctx+0(FP), CX - MOVQ 16(CX), DX - MOVQ 24(CX), BX - CMPQ BX, $0x04 - JB error_max_decoded_size_exceeded - LEAQ (DX)(BX*1), BX - MOVQ (CX), SI - MOVQ (SI), R8 - MOVQ 24(SI), R9 - MOVQ 32(SI), R10 - MOVBQZX 40(SI), R11 - MOVQ 32(CX), SI - MOVBQZX 8(CX), DI - JMP loop_condition - -main_loop: - // Check if we have room for 4 bytes in the output buffer - LEAQ 4(DX), CX - CMPQ CX, BX - JGE error_max_decoded_size_exceeded - - // Decode 4 values - CMPQ R11, $0x20 - JL bitReader_fillFast_1_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), CX - SHLXQ R11, CX, CX - ORQ CX, R10 - -bitReader_fillFast_1_end: - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AL - MOVBQZX CL, CX - ADDQ CX, R11 - SHLXQ CX, R10, R10 - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AH - MOVBQZX CL, CX - ADDQ CX, R11 - SHLXQ CX, R10, R10 - BSWAPL AX - CMPQ R11, $0x20 - JL bitReader_fillFast_2_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), CX - SHLXQ R11, CX, CX - ORQ CX, R10 - -bitReader_fillFast_2_end: - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AH - MOVBQZX CL, CX - ADDQ CX, R11 - SHLXQ CX, R10, R10 - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AL - MOVBQZX CL, CX - ADDQ CX, R11 - SHLXQ CX, R10, R10 - BSWAPL AX - - // Store the decoded values - MOVL AX, (DX) - ADDQ $0x04, DX - -loop_condition: - CMPQ R9, $0x08 - JGE main_loop - - // Update ctx structure - MOVQ ctx+0(FP), AX - SUBQ 16(AX), DX - MOVQ DX, 40(AX) - MOVQ (AX), AX - MOVQ R9, 24(AX) - MOVQ R10, 32(AX) - MOVB R11, 40(AX) - RET + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + ADDQ BX, SI + MOVQ 56(AX), DI + MOVQ 48(AX), R8 + MOVQ (AX), R10 + MOVQ 32(R10), R9 + MOVBQZX 40(R10), R10 - // Report error -error_max_decoded_size_exceeded: - MOVQ ctx+0(FP), AX - MOVQ $-1, CX - MOVQ CX, 40(AX) +outer_loop: + // Iterations allowed by the output: (limit - op) / 16 + MOVQ SI, R11 + SUBQ BX, R11 + JLE done + SHRQ $0x04, R11 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ DI, R12 + SUBQ R8, R12 + SHRQ $0x03, R12 + CMPQ R12, R11 + CMOVQCS R12, R11 + TESTQ R11, R11 + JZ done + IMUL3Q $0x0e, R11, R11 + ADDQ BX, R11 + +inner_loop: + // symbol 0 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, (BX) + + // symbol 1 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 1(BX) + + // symbol 2 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 2(BX) + + // symbol 3 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 3(BX) + + // symbol 4 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 4(BX) + + // symbol 5 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 5(BX) + + // symbol 6 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 6(BX) + + // symbol 7 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 7(BX) + + // symbol 8 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 8(BX) + + // symbol 9 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 9(BX) + + // symbol 10 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 10(BX) + + // symbol 11 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 11(BX) + + // symbol 12 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 12(BX) + + // symbol 13 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 13(BX) + + // Refill the whole bytes consumed from below the window + MOVQ R10, R12 + ANDQ $0x38, R12 + MOVQ -8(DI), R13 + SHRQ $0x01, R13 + ANDQ $0x07, R10 + MOVQ R12, R14 + XORQ $0x3f, R14 + SHRXQ R14, R13, R13 + SHLXQ R10, R13, R13 + ORQ R13, R9 + SHRQ $0x03, R12 + SUBQ R12, DI + ADDQ $0x0e, BX + CMPQ BX, R11 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), DI + MOVQ DI, 24(CX) + MOVQ R9, 32(CX) + MOVB R10, 40(CX) + SUBQ 16(AX), BX + MOVQ BX, 40(AX) RET --- a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || appengine || !gc || noasm // This file contains a generic implementation of Decoder.Decompress4X. package huff0 --- a/vendor/github.com/klauspost/compress/s2/encode_all.go +++ b/vendor/github.com/klauspost/compress/s2/encode_all.go @@ -981,7 +981,7 @@ cv = load64(src, s) continue } - } else if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) { + } else if repeat > 0 && uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) { base := s + checkRep // Extend back for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; { --- a/vendor/github.com/klauspost/compress/s2/encode_better.go +++ b/vendor/github.com/klauspost/compress/s2/encode_better.go @@ -86,12 +86,18 @@ // We initialize repeat to 0, so we never match on first attempt repeat := 0 + // Cap the adaptive skip, as encodeBetterBlockAsm and the snappy Go encoder + // below already do. Without a cap, a long stretch with no matches lets the + // skip grow geometrically until it strides past the start of the next + // compressible region and loses the matches there. + const maxSkip = 100 + for { candidateL := 0 nextS := 0 for { // Next src position to check - nextS = s + (s-nextEmit)>>7 + 1 + nextS = s + min((s-nextEmit)>>7+1, maxSkip) if nextS > sLimit { goto emitRemainder } --- a/vendor/github.com/klauspost/compress/s2/encode_go.go +++ b/vendor/github.com/klauspost/compress/s2/encode_go.go @@ -1,4 +1,4 @@ -//go:build !amd64 || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || appengine || !gc || noasm package s2 @@ -7,7 +7,7 @@ "math/bits" ) -const hasAmd64Asm = false +const hasAsm = false // encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It // assumes that the varint-encoded length of the decompressed bytes has already --- a/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s +++ b/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s @@ -1,6 +1,6 @@ -// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_amd64.go -pkg=s2. DO NOT EDIT. +// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_asm.go -pkg=s2. DO NOT EDIT. -//go:build !appengine && !noasm && gc && !noasm +//go:build (amd64 || arm64) && !appengine && !noasm && gc && !noasm #include "textflag.h" @@ -274,16 +274,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm matchlen_bsf_16repeat_extend_encodeBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm matchlen_match8_repeat_extend_encodeBlockAsm: CMPL R9, $0x08 @@ -296,16 +290,10 @@ JMP matchlen_match4_repeat_extend_encodeBlockAsm matchlen_bsf_8_repeat_extend_encodeBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm matchlen_match4_repeat_extend_encodeBlockAsm: CMPL R9, $0x04 @@ -898,16 +886,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBlockAsm matchlen_bsf_16match_nolit_encodeBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm matchlen_match8_match_nolit_encodeBlockAsm: CMPL DI, $0x08 @@ -920,16 +902,10 @@ JMP matchlen_match4_match_nolit_encodeBlockAsm matchlen_bsf_8_match_nolit_encodeBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm matchlen_match4_match_nolit_encodeBlockAsm: CMPL DI, $0x04 @@ -1689,16 +1665,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB matchlen_bsf_16repeat_extend_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm4MB + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm4MB matchlen_match8_repeat_extend_encodeBlockAsm4MB: CMPL R9, $0x08 @@ -1711,16 +1681,10 @@ JMP matchlen_match4_repeat_extend_encodeBlockAsm4MB matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm4MB + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm4MB matchlen_match4_repeat_extend_encodeBlockAsm4MB: CMPL R9, $0x04 @@ -2263,16 +2227,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBlockAsm4MB matchlen_bsf_16match_nolit_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm4MB + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm4MB matchlen_match8_match_nolit_encodeBlockAsm4MB: CMPL DI, $0x08 @@ -2285,16 +2243,10 @@ JMP matchlen_match4_match_nolit_encodeBlockAsm4MB matchlen_bsf_8_match_nolit_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm4MB + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm4MB matchlen_match4_match_nolit_encodeBlockAsm4MB: CMPL DI, $0x04 @@ -3006,16 +2958,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm12B matchlen_bsf_16repeat_extend_encodeBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm12B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm12B matchlen_match8_repeat_extend_encodeBlockAsm12B: CMPL R9, $0x08 @@ -3028,16 +2974,10 @@ JMP matchlen_match4_repeat_extend_encodeBlockAsm12B matchlen_bsf_8_repeat_extend_encodeBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm12B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm12B matchlen_match4_repeat_extend_encodeBlockAsm12B: CMPL R9, $0x04 @@ -3458,16 +3398,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBlockAsm12B matchlen_bsf_16match_nolit_encodeBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm12B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm12B matchlen_match8_match_nolit_encodeBlockAsm12B: CMPL DI, $0x08 @@ -3480,16 +3414,10 @@ JMP matchlen_match4_match_nolit_encodeBlockAsm12B matchlen_bsf_8_match_nolit_encodeBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm12B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm12B matchlen_match4_match_nolit_encodeBlockAsm12B: CMPL DI, $0x04 @@ -4091,16 +4019,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm10B matchlen_bsf_16repeat_extend_encodeBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm10B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm10B matchlen_match8_repeat_extend_encodeBlockAsm10B: CMPL R9, $0x08 @@ -4113,16 +4035,10 @@ JMP matchlen_match4_repeat_extend_encodeBlockAsm10B matchlen_bsf_8_repeat_extend_encodeBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm10B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm10B matchlen_match4_repeat_extend_encodeBlockAsm10B: CMPL R9, $0x04 @@ -4543,16 +4459,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBlockAsm10B matchlen_bsf_16match_nolit_encodeBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm10B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm10B matchlen_match8_match_nolit_encodeBlockAsm10B: CMPL DI, $0x08 @@ -4565,16 +4475,10 @@ JMP matchlen_match4_match_nolit_encodeBlockAsm10B matchlen_bsf_8_match_nolit_encodeBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm10B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm10B matchlen_match4_match_nolit_encodeBlockAsm10B: CMPL DI, $0x04 @@ -5176,16 +5080,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm8B matchlen_bsf_16repeat_extend_encodeBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm8B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm8B matchlen_match8_repeat_extend_encodeBlockAsm8B: CMPL R9, $0x08 @@ -5198,16 +5096,10 @@ JMP matchlen_match4_repeat_extend_encodeBlockAsm8B matchlen_bsf_8_repeat_extend_encodeBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm8B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm8B matchlen_match4_repeat_extend_encodeBlockAsm8B: CMPL R9, $0x04 @@ -5614,16 +5506,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBlockAsm8B matchlen_bsf_16match_nolit_encodeBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm8B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm8B matchlen_match8_match_nolit_encodeBlockAsm8B: CMPL DI, $0x08 @@ -5636,16 +5522,10 @@ JMP matchlen_match4_match_nolit_encodeBlockAsm8B matchlen_bsf_8_match_nolit_encodeBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm8B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm8B matchlen_match4_match_nolit_encodeBlockAsm8B: CMPL DI, $0x04 @@ -6140,16 +6020,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm matchlen_bsf_16match_nolit_encodeBetterBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm matchlen_match8_match_nolit_encodeBetterBlockAsm: CMPL R8, $0x08 @@ -6162,16 +6036,10 @@ JMP matchlen_match4_match_nolit_encodeBetterBlockAsm matchlen_bsf_8_match_nolit_encodeBetterBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm matchlen_match4_match_nolit_encodeBetterBlockAsm: CMPL R8, $0x04 @@ -7235,16 +7103,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm4MB + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm4MB matchlen_match8_match_nolit_encodeBetterBlockAsm4MB: CMPL R8, $0x08 @@ -7257,16 +7119,10 @@ JMP matchlen_match4_match_nolit_encodeBetterBlockAsm4MB matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm4MB + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm4MB matchlen_match4_match_nolit_encodeBetterBlockAsm4MB: CMPL R8, $0x04 @@ -8256,16 +8112,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm12B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm12B matchlen_match8_match_nolit_encodeBetterBlockAsm12B: CMPL R8, $0x08 @@ -8278,16 +8128,10 @@ JMP matchlen_match4_match_nolit_encodeBetterBlockAsm12B matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm12B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm12B matchlen_match4_match_nolit_encodeBetterBlockAsm12B: CMPL R8, $0x04 @@ -9130,16 +8974,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm10B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm10B matchlen_match8_match_nolit_encodeBetterBlockAsm10B: CMPL R8, $0x08 @@ -9152,16 +8990,10 @@ JMP matchlen_match4_match_nolit_encodeBetterBlockAsm10B matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm10B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm10B matchlen_match4_match_nolit_encodeBetterBlockAsm10B: CMPL R8, $0x04 @@ -10004,16 +9836,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm8B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm8B matchlen_match8_match_nolit_encodeBetterBlockAsm8B: CMPL R8, $0x08 @@ -10026,16 +9852,10 @@ JMP matchlen_match4_match_nolit_encodeBetterBlockAsm8B matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm8B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm8B matchlen_match4_match_nolit_encodeBetterBlockAsm8B: CMPL R8, $0x04 @@ -10985,16 +10805,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm matchlen_match8_repeat_extend_encodeSnappyBlockAsm: CMPL R8, $0x08 @@ -11007,16 +10821,10 @@ JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm matchlen_match4_repeat_extend_encodeSnappyBlockAsm: CMPL R8, $0x04 @@ -11329,16 +11137,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm matchlen_bsf_16match_nolit_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm matchlen_match8_match_nolit_encodeSnappyBlockAsm: CMPL DI, $0x08 @@ -11351,16 +11153,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm matchlen_match4_match_nolit_encodeSnappyBlockAsm: CMPL DI, $0x04 @@ -11901,16 +11697,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K: CMPL R8, $0x08 @@ -11923,16 +11713,10 @@ JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K: CMPL R8, $0x04 @@ -12205,16 +11989,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm64K + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm64K matchlen_match8_match_nolit_encodeSnappyBlockAsm64K: CMPL DI, $0x08 @@ -12227,16 +12005,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm64K matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm64K + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm64K matchlen_match4_match_nolit_encodeSnappyBlockAsm64K: CMPL DI, $0x04 @@ -12737,16 +12509,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B: CMPL R8, $0x08 @@ -12759,16 +12525,10 @@ JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B: CMPL R8, $0x04 @@ -13041,16 +12801,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm12B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm12B matchlen_match8_match_nolit_encodeSnappyBlockAsm12B: CMPL DI, $0x08 @@ -13063,16 +12817,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm12B matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm12B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm12B matchlen_match4_match_nolit_encodeSnappyBlockAsm12B: CMPL DI, $0x04 @@ -13573,16 +13321,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B: CMPL R8, $0x08 @@ -13595,16 +13337,10 @@ JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B: CMPL R8, $0x04 @@ -13877,16 +13613,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm10B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm10B matchlen_match8_match_nolit_encodeSnappyBlockAsm10B: CMPL DI, $0x08 @@ -13899,16 +13629,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm10B matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm10B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm10B matchlen_match4_match_nolit_encodeSnappyBlockAsm10B: CMPL DI, $0x04 @@ -14409,16 +14133,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B: CMPL R8, $0x08 @@ -14431,16 +14149,10 @@ JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B: CMPL R8, $0x04 @@ -14711,16 +14423,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm8B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm8B matchlen_match8_match_nolit_encodeSnappyBlockAsm8B: CMPL DI, $0x08 @@ -14733,16 +14439,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm8B matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm8B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm8B matchlen_match4_match_nolit_encodeSnappyBlockAsm8B: CMPL DI, $0x04 @@ -15144,16 +14844,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm: CMPL R8, $0x08 @@ -15166,16 +14860,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm: CMPL R8, $0x04 @@ -15790,16 +15478,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm64K + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm64K matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K: CMPL R8, $0x08 @@ -15812,16 +15494,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm64K + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm64K matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K: CMPL R8, $0x04 @@ -16371,16 +16047,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm12B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm12B matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B: CMPL R8, $0x08 @@ -16393,16 +16063,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm12B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm12B matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B: CMPL R8, $0x04 @@ -16952,16 +16616,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm10B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm10B matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B: CMPL R8, $0x08 @@ -16974,16 +16632,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm10B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm10B matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B: CMPL R8, $0x04 @@ -17533,16 +17185,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm8B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm8B matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B: CMPL R8, $0x08 @@ -17555,16 +17201,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm8B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm8B matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B: CMPL R8, $0x04 @@ -18138,16 +17778,10 @@ JMP matchlen_loopback_16_repeat_extend_calcBlockSize matchlen_bsf_16repeat_extend_calcBlockSize: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_calcBlockSize + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_calcBlockSize matchlen_match8_repeat_extend_calcBlockSize: CMPL R8, $0x08 @@ -18160,16 +17794,10 @@ JMP matchlen_match4_repeat_extend_calcBlockSize matchlen_bsf_8_repeat_extend_calcBlockSize: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_calcBlockSize + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_calcBlockSize matchlen_match4_repeat_extend_calcBlockSize: CMPL R8, $0x04 @@ -18371,16 +17999,10 @@ JMP matchlen_loopback_16_match_nolit_calcBlockSize matchlen_bsf_16match_nolit_calcBlockSize: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_calcBlockSize + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_calcBlockSize matchlen_match8_match_nolit_calcBlockSize: CMPL DI, $0x08 @@ -18393,16 +18015,10 @@ JMP matchlen_match4_match_nolit_calcBlockSize matchlen_bsf_8_match_nolit_calcBlockSize: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_calcBlockSize + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_calcBlockSize matchlen_match4_match_nolit_calcBlockSize: CMPL DI, $0x04 @@ -18723,16 +18339,10 @@ JMP matchlen_loopback_16_repeat_extend_calcBlockSizeSmall matchlen_bsf_16repeat_extend_calcBlockSizeSmall: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_calcBlockSizeSmall + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_calcBlockSizeSmall matchlen_match8_repeat_extend_calcBlockSizeSmall: CMPL R8, $0x08 @@ -18745,16 +18355,10 @@ JMP matchlen_match4_repeat_extend_calcBlockSizeSmall matchlen_bsf_8_repeat_extend_calcBlockSizeSmall: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_calcBlockSizeSmall + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_calcBlockSizeSmall matchlen_match4_repeat_extend_calcBlockSizeSmall: CMPL R8, $0x04 @@ -18926,16 +18530,10 @@ JMP matchlen_loopback_16_match_nolit_calcBlockSizeSmall matchlen_bsf_16match_nolit_calcBlockSizeSmall: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_calcBlockSizeSmall + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_calcBlockSizeSmall matchlen_match8_match_nolit_calcBlockSizeSmall: CMPL DI, $0x08 @@ -18948,16 +18546,10 @@ JMP matchlen_match4_match_nolit_calcBlockSizeSmall matchlen_bsf_8_match_nolit_calcBlockSizeSmall: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_calcBlockSizeSmall + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_calcBlockSizeSmall matchlen_match4_match_nolit_calcBlockSizeSmall: CMPL DI, $0x04 @@ -19737,16 +19329,10 @@ JMP matchlen_loopback_16_standalone matchlen_bsf_16standalone: -#ifdef GOAMD64_v3 TZCNTQ DI, DI - -#else - BSFQ DI, DI - -#endif - SARQ $0x03, DI - LEAL 8(SI)(DI*1), SI - JMP gen_match_len_end + SARQ $0x03, DI + LEAL 8(SI)(DI*1), SI + JMP gen_match_len_end matchlen_match8_standalone: CMPL DX, $0x08 @@ -19759,16 +19345,10 @@ JMP matchlen_match4_standalone matchlen_bsf_8_standalone: -#ifdef GOAMD64_v3 TZCNTQ BX, BX - -#else - BSFQ BX, BX - -#endif - SARQ $0x03, BX - LEAL (SI)(BX*1), SI - JMP gen_match_len_end + SARQ $0x03, BX + LEAL (SI)(BX*1), SI + JMP gen_match_len_end matchlen_match4_standalone: CMPL DX, $0x04 @@ -19802,7 +19382,7 @@ // func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) // Requires: SSE2 -TEXT ·cvtLZ4BlockAsm(SB), NOSPLIT, $0-64 +TEXT ·cvtLZ4BlockAsm(SB), NOSPLIT, $8-64 XORQ SI, SI MOVQ dst_base+0(FP), AX MOVQ dst_len+8(FP), CX @@ -19810,457 +19390,464 @@ MOVQ src_len+32(FP), BX LEAQ (DX)(BX*1), BX LEAQ -8(AX)(CX*1), CX - XORQ DI, DI + LEAQ -4(CX), DI + XORQ R8, R8 lz4_s2_loop: CMPQ DX, BX JAE lz4_s2_corrupt CMPQ AX, CX JAE lz4_s2_dstfull - MOVBQZX (DX), R8 - MOVQ R8, R9 - MOVQ R8, R10 - SHRQ $0x04, R9 - ANDQ $0x0f, R10 - CMPQ R8, $0xf0 + MOVBQZX (DX), R9 + MOVQ R9, R10 + MOVQ R9, R11 + SHRQ $0x04, R10 + ANDQ $0x0f, R11 + CMPQ R9, $0xf0 JB lz4_s2_ll_end lz4_s2_ll_loop: INCQ DX CMPQ DX, BX JAE lz4_s2_corrupt - MOVBQZX (DX), R8 - ADDQ R8, R9 - CMPQ R8, $0xff + MOVBQZX (DX), R9 + ADDQ R9, R10 + CMPQ R9, $0xff JEQ lz4_s2_ll_loop lz4_s2_ll_end: - LEAQ (DX)(R9*1), R8 - ADDQ $0x04, R10 - CMPQ R8, BX + LEAQ (DX)(R10*1), R9 + ADDQ $0x04, R11 + CMPQ R9, BX JAE lz4_s2_corrupt INCQ DX - INCQ R8 - TESTQ R9, R9 + INCQ R9 + TESTQ R10, R10 JZ lz4_s2_lits_done - LEAQ (AX)(R9*1), R11 - CMPQ R11, CX + LEAQ (AX)(R10*1), R12 + CMPQ R12, DI JAE lz4_s2_dstfull - ADDQ R9, SI - LEAL -1(R9), R11 - CMPL R11, $0x3c + ADDQ R10, SI + LEAL -1(R10), R12 + CMPL R12, $0x3c JB one_byte_lz4_s2 - CMPL R11, $0x00000100 + CMPL R12, $0x00000100 JB two_bytes_lz4_s2 - CMPL R11, $0x00010000 + CMPL R12, $0x00010000 JB three_bytes_lz4_s2 - CMPL R11, $0x01000000 + CMPL R12, $0x01000000 JB four_bytes_lz4_s2 MOVB $0xfc, (AX) - MOVL R11, 1(AX) + MOVL R12, 1(AX) ADDQ $0x05, AX JMP memmove_long_lz4_s2 four_bytes_lz4_s2: - MOVL R11, R12 - SHRL $0x10, R12 + MOVL R12, R13 + SHRL $0x10, R13 MOVB $0xf8, (AX) - MOVW R11, 1(AX) - MOVB R12, 3(AX) + MOVW R12, 1(AX) + MOVB R13, 3(AX) ADDQ $0x04, AX JMP memmove_long_lz4_s2 three_bytes_lz4_s2: MOVB $0xf4, (AX) - MOVW R11, 1(AX) + MOVW R12, 1(AX) ADDQ $0x03, AX JMP memmove_long_lz4_s2 two_bytes_lz4_s2: MOVB $0xf0, (AX) - MOVB R11, 1(AX) + MOVB R12, 1(AX) ADDQ $0x02, AX - CMPL R11, $0x40 + CMPL R12, $0x40 JB memmove_lz4_s2 JMP memmove_long_lz4_s2 one_byte_lz4_s2: - SHLB $0x02, R11 - MOVB R11, (AX) + SHLB $0x02, R12 + MOVB R12, (AX) ADDQ $0x01, AX memmove_lz4_s2: - LEAQ (AX)(R9*1), R11 + LEAQ (AX)(R10*1), R12 // genMemMoveShort - CMPQ R9, $0x08 + CMPQ R10, $0x08 JBE emit_lit_memmove_lz4_s2_memmove_move_8 - CMPQ R9, $0x10 + CMPQ R10, $0x10 JBE emit_lit_memmove_lz4_s2_memmove_move_8through16 - CMPQ R9, $0x20 + CMPQ R10, $0x20 JBE emit_lit_memmove_lz4_s2_memmove_move_17through32 JMP emit_lit_memmove_lz4_s2_memmove_move_33through64 emit_lit_memmove_lz4_s2_memmove_move_8: - MOVQ (DX), R12 - MOVQ R12, (AX) + MOVQ (DX), R13 + MOVQ R13, (AX) JMP memmove_end_copy_lz4_s2 emit_lit_memmove_lz4_s2_memmove_move_8through16: - MOVQ (DX), R12 - MOVQ -8(DX)(R9*1), DX - MOVQ R12, (AX) - MOVQ DX, -8(AX)(R9*1) + MOVQ (DX), R13 + MOVQ -8(DX)(R10*1), DX + MOVQ R13, (AX) + MOVQ DX, -8(AX)(R10*1) JMP memmove_end_copy_lz4_s2 emit_lit_memmove_lz4_s2_memmove_move_17through32: MOVOU (DX), X0 - MOVOU -16(DX)(R9*1), X1 + MOVOU -16(DX)(R10*1), X1 MOVOU X0, (AX) - MOVOU X1, -16(AX)(R9*1) + MOVOU X1, -16(AX)(R10*1) JMP memmove_end_copy_lz4_s2 emit_lit_memmove_lz4_s2_memmove_move_33through64: MOVOU (DX), X0 MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 + MOVOU -32(DX)(R10*1), X2 + MOVOU -16(DX)(R10*1), X3 MOVOU X0, (AX) MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) + MOVOU X2, -32(AX)(R10*1) + MOVOU X3, -16(AX)(R10*1) memmove_end_copy_lz4_s2: - MOVQ R11, AX + MOVQ R12, AX JMP lz4_s2_lits_emit_done memmove_long_lz4_s2: - LEAQ (AX)(R9*1), R11 + LEAQ (AX)(R10*1), R12 // genMemMoveLong MOVOU (DX), X0 MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ AX, R12 - ANDL $0x0000001f, R12 - MOVQ $0x00000040, R14 - SUBQ R12, R14 - DECQ R13 + MOVOU -32(DX)(R10*1), X2 + MOVOU -16(DX)(R10*1), X3 + MOVQ R10, R14 + SHRQ $0x05, R14 + MOVQ AX, R13 + ANDL $0x0000001f, R13 + MOVQ $0x00000040, R15 + SUBQ R13, R15 + DECQ R14 JA emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32 - LEAQ -32(DX)(R14*1), R12 - LEAQ -32(AX)(R14*1), R15 + LEAQ -32(DX)(R15*1), R13 + LEAQ -32(AX)(R15*1), BP emit_lit_memmove_long_lz4_s2large_big_loop_back: - MOVOU (R12), X4 - MOVOU 16(R12), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) + MOVOU (R13), X4 + MOVOU 16(R13), X5 + MOVOA X4, (BP) + MOVOA X5, 16(BP) + ADDQ $0x20, BP + ADDQ $0x20, R13 ADDQ $0x20, R15 - ADDQ $0x20, R12 - ADDQ $0x20, R14 - DECQ R13 + DECQ R14 JNA emit_lit_memmove_long_lz4_s2large_big_loop_back emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32: - MOVOU -32(DX)(R14*1), X4 - MOVOU -16(DX)(R14*1), X5 - MOVOA X4, -32(AX)(R14*1) - MOVOA X5, -16(AX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 + MOVOU -32(DX)(R15*1), X4 + MOVOU -16(DX)(R15*1), X5 + MOVOA X4, -32(AX)(R15*1) + MOVOA X5, -16(AX)(R15*1) + ADDQ $0x20, R15 + CMPQ R10, R15 JAE emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32 MOVOU X0, (AX) MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) - MOVQ R11, AX + MOVOU X2, -32(AX)(R10*1) + MOVOU X3, -16(AX)(R10*1) + MOVQ R12, AX lz4_s2_lits_emit_done: - MOVQ R8, DX + MOVQ R9, DX lz4_s2_lits_done: CMPQ DX, BX JNE lz4_s2_match - CMPQ R10, $0x04 + CMPQ R11, $0x04 JEQ lz4_s2_done JMP lz4_s2_corrupt lz4_s2_match: - LEAQ 2(DX), R8 - CMPQ R8, BX + LEAQ 2(DX), R9 + CMPQ R9, BX JAE lz4_s2_corrupt - MOVWQZX (DX), R9 - MOVQ R8, DX - TESTQ R9, R9 + MOVWQZX (DX), R10 + MOVQ R9, DX + TESTQ R10, R10 JZ lz4_s2_corrupt - CMPQ R9, SI + CMPQ R10, SI JA lz4_s2_corrupt - CMPQ R10, $0x13 + CMPQ R11, $0x13 JNE lz4_s2_ml_done lz4_s2_ml_loop: - MOVBQZX (DX), R8 + MOVBQZX (DX), R9 INCQ DX - ADDQ R8, R10 + ADDQ R9, R11 CMPQ DX, BX JAE lz4_s2_corrupt - CMPQ R8, $0xff + CMPQ R9, $0xff JEQ lz4_s2_ml_loop lz4_s2_ml_done: - ADDQ R10, SI - CMPQ R9, DI + ADDQ R11, SI + CMPQ R10, R8 JNE lz4_s2_docopy // emitRepeat emit_repeat_again_lz4_s2: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2 - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2 cant_repeat_two_offset_lz4_s2: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2 - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2 - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2 - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4_s2_loop JMP emit_repeat_again_lz4_s2 repeat_five_lz4_s2: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4_s2_loop repeat_four_lz4_s2: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4_s2_loop repeat_three_lz4_s2: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4_s2_loop repeat_two_lz4_s2: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop repeat_two_offset_lz4_s2: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop lz4_s2_docopy: - MOVQ R9, DI + MOVQ R10, R8 // emitCopy - CMPL R10, $0x40 + CMPL R11, $0x40 JBE two_byte_offset_short_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JAE long_offset_short_lz4_s2 - MOVL $0x00000001, R8 - LEAL 16(R8), R8 - MOVB R9, 1(AX) - MOVL R9, R11 - SHRL $0x08, R11 - SHLL $0x05, R11 - ORL R11, R8 - MOVB R8, (AX) + MOVL $0x00000001, R9 + LEAL 16(R9), R9 + MOVB R10, 1(AX) + MOVL R10, R12 + SHRL $0x08, R12 + SHLL $0x05, R12 + ORL R12, R9 + MOVB R9, (AX) ADDQ $0x02, AX - SUBL $0x08, R10 + SUBL $0x08, R11 // emitRepeat - LEAL -4(R10), R10 + LEAL -4(R11), R11 JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b emit_repeat_again_lz4_s2_emit_copy_short_2b: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2_emit_copy_short_2b - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2_emit_copy_short_2b - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2_emit_copy_short_2b cant_repeat_two_offset_lz4_s2_emit_copy_short_2b: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2_emit_copy_short_2b - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2_emit_copy_short_2b - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2_emit_copy_short_2b - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4_s2_loop JMP emit_repeat_again_lz4_s2_emit_copy_short_2b repeat_five_lz4_s2_emit_copy_short_2b: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4_s2_loop repeat_four_lz4_s2_emit_copy_short_2b: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4_s2_loop repeat_three_lz4_s2_emit_copy_short_2b: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4_s2_loop repeat_two_lz4_s2_emit_copy_short_2b: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop repeat_two_offset_lz4_s2_emit_copy_short_2b: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop long_offset_short_lz4_s2: MOVB $0xee, (AX) - MOVW R9, 1(AX) - LEAL -60(R10), R10 + MOVW R10, 1(AX) + LEAL -60(R11), R11 ADDQ $0x03, AX // emitRepeat emit_repeat_again_lz4_s2_emit_copy_short: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2_emit_copy_short - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2_emit_copy_short - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2_emit_copy_short cant_repeat_two_offset_lz4_s2_emit_copy_short: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2_emit_copy_short - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2_emit_copy_short - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2_emit_copy_short - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4_s2_loop JMP emit_repeat_again_lz4_s2_emit_copy_short repeat_five_lz4_s2_emit_copy_short: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4_s2_loop repeat_four_lz4_s2_emit_copy_short: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4_s2_loop repeat_three_lz4_s2_emit_copy_short: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4_s2_loop repeat_two_lz4_s2_emit_copy_short: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop repeat_two_offset_lz4_s2_emit_copy_short: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop two_byte_offset_short_lz4_s2: - MOVL R10, R8 - SHLL $0x02, R8 - CMPL R10, $0x0c + MOVL R11, R9 + SHLL $0x02, R9 + CMPL R11, $0x0c JAE emit_copy_three_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JAE emit_copy_three_lz4_s2 - LEAL -15(R8), R8 - MOVB R9, 1(AX) - SHRL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R8 - MOVB R8, (AX) + LEAL -15(R9), R9 + MOVB R10, 1(AX) + SHRL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R9 + MOVB R9, (AX) ADDQ $0x02, AX JMP lz4_s2_loop emit_copy_three_lz4_s2: - LEAL -2(R8), R8 - MOVB R8, (AX) - MOVW R9, 1(AX) + LEAL -2(R9), R9 + MOVB R9, (AX) + MOVW R10, 1(AX) ADDQ $0x03, AX JMP lz4_s2_loop @@ -20285,7 +19872,7 @@ // func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) // Requires: SSE2 -TEXT ·cvtLZ4sBlockAsm(SB), NOSPLIT, $0-64 +TEXT ·cvtLZ4sBlockAsm(SB), NOSPLIT, $8-64 XORQ SI, SI MOVQ dst_base+0(FP), AX MOVQ dst_len+8(FP), CX @@ -20293,459 +19880,466 @@ MOVQ src_len+32(FP), BX LEAQ (DX)(BX*1), BX LEAQ -8(AX)(CX*1), CX - XORQ DI, DI + LEAQ -4(CX), DI + XORQ R8, R8 lz4s_s2_loop: CMPQ DX, BX JAE lz4s_s2_corrupt CMPQ AX, CX JAE lz4s_s2_dstfull - MOVBQZX (DX), R8 - MOVQ R8, R9 - MOVQ R8, R10 - SHRQ $0x04, R9 - ANDQ $0x0f, R10 - CMPQ R8, $0xf0 + MOVBQZX (DX), R9 + MOVQ R9, R10 + MOVQ R9, R11 + SHRQ $0x04, R10 + ANDQ $0x0f, R11 + CMPQ R9, $0xf0 JB lz4s_s2_ll_end lz4s_s2_ll_loop: INCQ DX CMPQ DX, BX JAE lz4s_s2_corrupt - MOVBQZX (DX), R8 - ADDQ R8, R9 - CMPQ R8, $0xff + MOVBQZX (DX), R9 + ADDQ R9, R10 + CMPQ R9, $0xff JEQ lz4s_s2_ll_loop lz4s_s2_ll_end: - LEAQ (DX)(R9*1), R8 - ADDQ $0x03, R10 - CMPQ R8, BX + LEAQ (DX)(R10*1), R9 + ADDQ $0x03, R11 + CMPQ R9, BX JAE lz4s_s2_corrupt INCQ DX - INCQ R8 - TESTQ R9, R9 + INCQ R9 + TESTQ R10, R10 JZ lz4s_s2_lits_done - LEAQ (AX)(R9*1), R11 - CMPQ R11, CX + LEAQ (AX)(R10*1), R12 + CMPQ R12, DI JAE lz4s_s2_dstfull - ADDQ R9, SI - LEAL -1(R9), R11 - CMPL R11, $0x3c + ADDQ R10, SI + LEAL -1(R10), R12 + CMPL R12, $0x3c JB one_byte_lz4s_s2 - CMPL R11, $0x00000100 + CMPL R12, $0x00000100 JB two_bytes_lz4s_s2 - CMPL R11, $0x00010000 + CMPL R12, $0x00010000 JB three_bytes_lz4s_s2 - CMPL R11, $0x01000000 + CMPL R12, $0x01000000 JB four_bytes_lz4s_s2 MOVB $0xfc, (AX) - MOVL R11, 1(AX) + MOVL R12, 1(AX) ADDQ $0x05, AX JMP memmove_long_lz4s_s2 four_bytes_lz4s_s2: - MOVL R11, R12 - SHRL $0x10, R12 + MOVL R12, R13 + SHRL $0x10, R13 MOVB $0xf8, (AX) - MOVW R11, 1(AX) - MOVB R12, 3(AX) + MOVW R12, 1(AX) + MOVB R13, 3(AX) ADDQ $0x04, AX JMP memmove_long_lz4s_s2 three_bytes_lz4s_s2: MOVB $0xf4, (AX) - MOVW R11, 1(AX) + MOVW R12, 1(AX) ADDQ $0x03, AX JMP memmove_long_lz4s_s2 two_bytes_lz4s_s2: MOVB $0xf0, (AX) - MOVB R11, 1(AX) + MOVB R12, 1(AX) ADDQ $0x02, AX - CMPL R11, $0x40 + CMPL R12, $0x40 JB memmove_lz4s_s2 JMP memmove_long_lz4s_s2 one_byte_lz4s_s2: - SHLB $0x02, R11 - MOVB R11, (AX) + SHLB $0x02, R12 + MOVB R12, (AX) ADDQ $0x01, AX memmove_lz4s_s2: - LEAQ (AX)(R9*1), R11 + LEAQ (AX)(R10*1), R12 // genMemMoveShort - CMPQ R9, $0x08 + CMPQ R10, $0x08 JBE emit_lit_memmove_lz4s_s2_memmove_move_8 - CMPQ R9, $0x10 + CMPQ R10, $0x10 JBE emit_lit_memmove_lz4s_s2_memmove_move_8through16 - CMPQ R9, $0x20 + CMPQ R10, $0x20 JBE emit_lit_memmove_lz4s_s2_memmove_move_17through32 JMP emit_lit_memmove_lz4s_s2_memmove_move_33through64 emit_lit_memmove_lz4s_s2_memmove_move_8: - MOVQ (DX), R12 - MOVQ R12, (AX) + MOVQ (DX), R13 + MOVQ R13, (AX) JMP memmove_end_copy_lz4s_s2 emit_lit_memmove_lz4s_s2_memmove_move_8through16: - MOVQ (DX), R12 - MOVQ -8(DX)(R9*1), DX - MOVQ R12, (AX) - MOVQ DX, -8(AX)(R9*1) + MOVQ (DX), R13 + MOVQ -8(DX)(R10*1), DX + MOVQ R13, (AX) + MOVQ DX, -8(AX)(R10*1) JMP memmove_end_copy_lz4s_s2 emit_lit_memmove_lz4s_s2_memmove_move_17through32: MOVOU (DX), X0 - MOVOU -16(DX)(R9*1), X1 + MOVOU -16(DX)(R10*1), X1 MOVOU X0, (AX) - MOVOU X1, -16(AX)(R9*1) + MOVOU X1, -16(AX)(R10*1) JMP memmove_end_copy_lz4s_s2 emit_lit_memmove_lz4s_s2_memmove_move_33through64: MOVOU (DX), X0 MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 + MOVOU -32(DX)(R10*1), X2 + MOVOU -16(DX)(R10*1), X3 MOVOU X0, (AX) MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) + MOVOU X2, -32(AX)(R10*1) + MOVOU X3, -16(AX)(R10*1) memmove_end_copy_lz4s_s2: - MOVQ R11, AX + MOVQ R12, AX JMP lz4s_s2_lits_emit_done memmove_long_lz4s_s2: - LEAQ (AX)(R9*1), R11 + LEAQ (AX)(R10*1), R12 // genMemMoveLong MOVOU (DX), X0 MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ AX, R12 - ANDL $0x0000001f, R12 - MOVQ $0x00000040, R14 - SUBQ R12, R14 - DECQ R13 + MOVOU -32(DX)(R10*1), X2 + MOVOU -16(DX)(R10*1), X3 + MOVQ R10, R14 + SHRQ $0x05, R14 + MOVQ AX, R13 + ANDL $0x0000001f, R13 + MOVQ $0x00000040, R15 + SUBQ R13, R15 + DECQ R14 JA emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32 - LEAQ -32(DX)(R14*1), R12 - LEAQ -32(AX)(R14*1), R15 + LEAQ -32(DX)(R15*1), R13 + LEAQ -32(AX)(R15*1), BP emit_lit_memmove_long_lz4s_s2large_big_loop_back: - MOVOU (R12), X4 - MOVOU 16(R12), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) + MOVOU (R13), X4 + MOVOU 16(R13), X5 + MOVOA X4, (BP) + MOVOA X5, 16(BP) + ADDQ $0x20, BP + ADDQ $0x20, R13 ADDQ $0x20, R15 - ADDQ $0x20, R12 - ADDQ $0x20, R14 - DECQ R13 + DECQ R14 JNA emit_lit_memmove_long_lz4s_s2large_big_loop_back emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32: - MOVOU -32(DX)(R14*1), X4 - MOVOU -16(DX)(R14*1), X5 - MOVOA X4, -32(AX)(R14*1) - MOVOA X5, -16(AX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 + MOVOU -32(DX)(R15*1), X4 + MOVOU -16(DX)(R15*1), X5 + MOVOA X4, -32(AX)(R15*1) + MOVOA X5, -16(AX)(R15*1) + ADDQ $0x20, R15 + CMPQ R10, R15 JAE emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32 MOVOU X0, (AX) MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) - MOVQ R11, AX + MOVOU X2, -32(AX)(R10*1) + MOVOU X3, -16(AX)(R10*1) + MOVQ R12, AX lz4s_s2_lits_emit_done: - MOVQ R8, DX + MOVQ R9, DX lz4s_s2_lits_done: CMPQ DX, BX JNE lz4s_s2_match - CMPQ R10, $0x03 + CMPQ R11, $0x03 JEQ lz4s_s2_done JMP lz4s_s2_corrupt lz4s_s2_match: - CMPQ R10, $0x03 + CMPQ R11, $0x03 JEQ lz4s_s2_loop - LEAQ 2(DX), R8 - CMPQ R8, BX + LEAQ 2(DX), R9 + CMPQ R9, BX JAE lz4s_s2_corrupt - MOVWQZX (DX), R9 - MOVQ R8, DX - TESTQ R9, R9 + MOVWQZX (DX), R10 + MOVQ R9, DX + TESTQ R10, R10 JZ lz4s_s2_corrupt - CMPQ R9, SI + CMPQ R10, SI JA lz4s_s2_corrupt - CMPQ R10, $0x12 + CMPQ R11, $0x12 JNE lz4s_s2_ml_done lz4s_s2_ml_loop: - MOVBQZX (DX), R8 + MOVBQZX (DX), R9 INCQ DX - ADDQ R8, R10 + ADDQ R9, R11 CMPQ DX, BX JAE lz4s_s2_corrupt - CMPQ R8, $0xff + CMPQ R9, $0xff JEQ lz4s_s2_ml_loop lz4s_s2_ml_done: - ADDQ R10, SI - CMPQ R9, DI + ADDQ R11, SI + CMPQ R10, R8 JNE lz4s_s2_docopy // emitRepeat emit_repeat_again_lz4_s2: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2 - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2 cant_repeat_two_offset_lz4_s2: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2 - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2 - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2 - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4s_s2_loop JMP emit_repeat_again_lz4_s2 repeat_five_lz4_s2: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4s_s2_loop repeat_four_lz4_s2: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4s_s2_loop repeat_three_lz4_s2: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4s_s2_loop repeat_two_lz4_s2: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop repeat_two_offset_lz4_s2: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop lz4s_s2_docopy: - MOVQ R9, DI + MOVQ R10, R8 // emitCopy - CMPL R10, $0x40 + CMPL R11, $0x40 JBE two_byte_offset_short_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JAE long_offset_short_lz4_s2 - MOVL $0x00000001, R8 - LEAL 16(R8), R8 - MOVB R9, 1(AX) - MOVL R9, R11 - SHRL $0x08, R11 - SHLL $0x05, R11 - ORL R11, R8 - MOVB R8, (AX) + MOVL $0x00000001, R9 + LEAL 16(R9), R9 + MOVB R10, 1(AX) + MOVL R10, R12 + SHRL $0x08, R12 + SHLL $0x05, R12 + ORL R12, R9 + MOVB R9, (AX) ADDQ $0x02, AX - SUBL $0x08, R10 + SUBL $0x08, R11 // emitRepeat - LEAL -4(R10), R10 + LEAL -4(R11), R11 JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b emit_repeat_again_lz4_s2_emit_copy_short_2b: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2_emit_copy_short_2b - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2_emit_copy_short_2b - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2_emit_copy_short_2b cant_repeat_two_offset_lz4_s2_emit_copy_short_2b: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2_emit_copy_short_2b - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2_emit_copy_short_2b - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2_emit_copy_short_2b - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4s_s2_loop JMP emit_repeat_again_lz4_s2_emit_copy_short_2b repeat_five_lz4_s2_emit_copy_short_2b: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4s_s2_loop repeat_four_lz4_s2_emit_copy_short_2b: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4s_s2_loop repeat_three_lz4_s2_emit_copy_short_2b: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4s_s2_loop repeat_two_lz4_s2_emit_copy_short_2b: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop repeat_two_offset_lz4_s2_emit_copy_short_2b: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop long_offset_short_lz4_s2: MOVB $0xee, (AX) - MOVW R9, 1(AX) - LEAL -60(R10), R10 + MOVW R10, 1(AX) + LEAL -60(R11), R11 ADDQ $0x03, AX // emitRepeat emit_repeat_again_lz4_s2_emit_copy_short: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2_emit_copy_short - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2_emit_copy_short - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2_emit_copy_short cant_repeat_two_offset_lz4_s2_emit_copy_short: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2_emit_copy_short - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2_emit_copy_short - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2_emit_copy_short - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4s_s2_loop JMP emit_repeat_again_lz4_s2_emit_copy_short repeat_five_lz4_s2_emit_copy_short: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4s_s2_loop repeat_four_lz4_s2_emit_copy_short: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4s_s2_loop repeat_three_lz4_s2_emit_copy_short: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4s_s2_loop repeat_two_lz4_s2_emit_copy_short: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop repeat_two_offset_lz4_s2_emit_copy_short: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop two_byte_offset_short_lz4_s2: - MOVL R10, R8 - SHLL $0x02, R8 - CMPL R10, $0x0c + MOVL R11, R9 + SHLL $0x02, R9 + CMPL R11, $0x0c JAE emit_copy_three_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JAE emit_copy_three_lz4_s2 - LEAL -15(R8), R8 - MOVB R9, 1(AX) - SHRL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R8 - MOVB R8, (AX) + LEAL -15(R9), R9 + MOVB R10, 1(AX) + SHRL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R9 + MOVB R9, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop emit_copy_three_lz4_s2: - LEAL -2(R8), R8 - MOVB R8, (AX) - MOVW R9, 1(AX) + LEAL -2(R9), R9 + MOVB R9, (AX) + MOVW R10, 1(AX) ADDQ $0x03, AX JMP lz4s_s2_loop --- a/vendor/github.com/klauspost/compress/s2/index.go +++ b/vendor/github.com/klauspost/compress/s2/index.go @@ -151,7 +151,7 @@ i.estBlockUncomp += i.estBlockUncomp * int64(removeN) } -func (i *Index) appendTo(b []byte, uncompTotal, compTotal int64) []byte { +func (i *Index) appendTo(b []byte, compTotal, uncompTotal int64) []byte { i.reduce() var tmp [binary.MaxVarintLen64]byte @@ -425,7 +425,7 @@ _, err := io.ReadFull(r, buf[:4]) if err != nil { if err == io.EOF { - return i.appendTo(nil, i.TotalUncompressed, i.TotalCompressed), nil + return i.appendTo(nil, i.TotalCompressed, i.TotalUncompressed), nil } return nil, err } --- a/vendor/github.com/klauspost/compress/s2/lz4convert.go +++ b/vendor/github.com/klauspost/compress/s2/lz4convert.go @@ -13,6 +13,7 @@ // LZ4Converter provides conversion from LZ4 blocks as defined here: // https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md type LZ4Converter struct { + noAsm bool } // ErrDstTooSmall is returned when provided destination is too small. @@ -32,7 +33,7 @@ s, d := 0, len(dst) dst = dst[:cap(dst)] - if !debug && hasAmd64Asm { + if !debug && hasAsm && !l.noAsm { res, sz := cvtLZ4BlockAsm(dst[d:], src) if res < 0 { const ( @@ -96,7 +97,8 @@ } s++ if ll > 0 { - if d+ll > dLimit { + // Leave room for a 5-byte literal header and an 8-byte copy. + if d+ll > len(dst)-13 { return nil, 0, ErrDstTooSmall } if debug { @@ -209,11 +211,13 @@ dst[2] = uint8(length >> 0) dst[1] = 0 dst[0] = 7<<2 | tagCopy1 + d += 5 if left > 0 { - d += 5 + emitRepeat16(dst[5:], offset, left) - break + // The loop condition checks room for the next chunk. + dst = dst[5:] + length = left + continue } - d += 5 break } } @@ -288,7 +292,7 @@ s, d := 0, len(dst) dst = dst[:cap(dst)] // Use assembly when possible - if !debug && hasAmd64Asm { + if !debug && hasAsm && !l.noAsm { res, sz := cvtLZ4BlockSnappyAsm(dst[d:], src) if res < 0 { const ( @@ -488,6 +492,11 @@ length -= 1 << 16 left := 0 if length > maxRepeat { + if len(dst) < 10 { + // No room for this and the next chunk. + // Report more than len(dst) so the caller sees dst is full. + return len(dst) + 1 + } left = length - maxRepeat + 4 length = maxRepeat - 4 } --- a/vendor/github.com/klauspost/compress/s2/lz4sconvert.go +++ b/vendor/github.com/klauspost/compress/s2/lz4sconvert.go @@ -21,6 +21,7 @@ // For Min Match of 4 bytes, Copy length value 1-15 means length 4-18 with 18 bytes adding an extra byte. // ONLY "Min match of 4 bytes" is supported. type LZ4sConverter struct { + noAsm bool } // ConvertBlock will convert an LZ4s block and append it as an S2 @@ -37,7 +38,7 @@ s, d := 0, len(dst) dst = dst[:cap(dst)] - if !debug && hasAmd64Asm { + if !debug && hasAsm && !l.noAsm { res, sz := cvtLZ4sBlockAsm(dst[d:], src) if res < 0 { const ( @@ -101,7 +102,8 @@ } s++ if ll > 0 { - if d+ll > dLimit { + // Leave room for a 5-byte literal header and an 8-byte copy. + if d+ll > len(dst)-13 { return nil, 0, ErrDstTooSmall } if debug { @@ -218,11 +220,13 @@ dst[2] = uint8(length >> 0) dst[1] = 0 dst[0] = 7<<2 | tagCopy1 + d += 5 if left > 0 { - d += 5 + emitRepeat16(dst[5:], offset, left) - break + // The loop condition checks room for the next chunk. + dst = dst[5:] + length = left + continue } - d += 5 break } } @@ -297,7 +301,7 @@ s, d := 0, len(dst) dst = dst[:cap(dst)] // Use assembly when possible - if !debug && hasAmd64Asm { + if !debug && hasAsm && !l.noAsm { res, sz := cvtLZ4sBlockSnappyAsm(dst[d:], src) if res < 0 { const ( --- a/vendor/github.com/klauspost/compress/s2/reader.go +++ b/vendor/github.com/klauspost/compress/s2/reader.go @@ -216,13 +216,18 @@ return r.err == nil } if rs, ok := r.r.(io.ReadSeeker); ok { - _, err := rs.Seek(int64(n), io.SeekCurrent) - if err == nil { - return true - } - if err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) { - r.err = ErrCorrupt - return false + if cur, err := rs.Seek(0, io.SeekCurrent); err == nil { + if end, err := rs.Seek(0, io.SeekEnd); err == nil { + if cur+int64(n) <= end { + if _, err := rs.Seek(cur+int64(n), io.SeekStart); err == nil { + return true + } + } + if _, err := rs.Seek(cur, io.SeekStart); err != nil { + r.err = ErrCorrupt + return false + } + } } } for n > 0 { @@ -446,9 +451,7 @@ } // Writer var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { for toWrite := range queue { entry := <-toWrite reUse <- toWrite @@ -475,7 +478,7 @@ } aWritten += int64(n) } - }() + }) defer func() { if r.err != nil { --- a/vendor/github.com/klauspost/compress/s2/writer.go +++ b/vendor/github.com/klauspost/compress/s2/writer.go @@ -139,11 +139,9 @@ toWrite := make(chan chan result, w.concurrency) w.output = toWrite - w.writerWg.Add(1) // Start a writer goroutine that will write all output in order. - go func() { - defer w.writerWg.Done() + w.writerWg.Go(func() { // Get a queued write. for write := range toWrite { @@ -175,7 +173,7 @@ // This can be used for synchronizing flushes. close(write) } - }() + }) } // Write satisfies the io.Writer interface. @@ -811,7 +809,7 @@ if w.pad <= 1 { compSize = w.written } - index = w.index.appendTo(w.ibuf[:0], w.uncompWritten, compSize) + index = w.index.appendTo(w.ibuf[:0], compSize, w.uncompWritten) // Count as written for padding. if w.appendIndex { w.written += int64(len(index)) --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -240,9 +240,11 @@ b.dst[i] = v } hist.appendKeep(b.dst) + hist.decoders.consumeSyncLen(len(b.dst)) return nil case blockTypeRaw: hist.appendKeep(b.data) + hist.decoders.consumeSyncLen(len(b.data)) return nil case blockTypeCompressed: saved := b.dst @@ -400,8 +402,9 @@ } } var err error - // Use our out buffer. - huff.MaxDecodedSize = litRegenSize + // Decoder.Decompress* uses cap(dst) for the size limit. Do not write + // MaxDecodedSize on hist.huffTree: with a trained dictionary that + // pointer aliases the shared dict.litEnc and concurrent DecodeAll races. if fourStreams { literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) } else { @@ -486,6 +489,7 @@ } if hist.decoders.nSeqs == 0 { b.dst = append(b.dst, hist.decoders.literals...) + hist.decoders.consumeSyncLen(len(hist.decoders.literals)) return nil } before := len(hist.decoders.out) @@ -673,10 +677,6 @@ hbytes := hist.b if len(hbytes) > hist.windowSize { hbytes = hbytes[len(hbytes)-hist.windowSize:] - // We do not need history anymore. - if hist.dict != nil { - hist.dict.content = nil - } } hist.decoders.windowSize = hist.windowSize hist.decoders.out = b.dst[:0] --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -7,8 +7,11 @@ import ( "context" "encoding/binary" + "errors" "io" + "maps" "sync" + "sync/atomic" "github.com/klauspost/compress/zstd/internal/xxhash" ) @@ -20,7 +23,8 @@ // A decoder can safely be re-used even if the previous stream failed. // To release the resources, you must call the Close() function on a decoder. type Decoder struct { - o decoderOptions + // publishedOptions contains the current immutable decoder options. + publishedOptions atomic.Pointer[decoderOptions] // Unreferenced decoders, ready for use. decoders chan *blockDec @@ -84,9 +88,10 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { initPredefined() var d Decoder - d.o.setDefault() - for _, o := range opts { - err := o(&d.o) + var o decoderOptions + o.setDefault() + for _, opt := range opts { + err := opt(&o) if err != nil { return nil, err } @@ -99,17 +104,19 @@ } // Initialize dict map if needed. - if d.o.dicts == nil { - d.o.dicts = make(map[uint32]*dict) + if o.dicts == nil { + o.dicts = make(map[uint32]*dict) } // Create decoders - d.decoders = make(chan *blockDec, d.o.concurrent) - for i := 0; i < d.o.concurrent; i++ { - dec := newBlockDec(d.o.lowMem) - dec.localFrame = newFrameDec(d.o) + d.decoders = make(chan *blockDec, o.concurrent) + for i := 0; i < o.concurrent; i++ { + dec := newBlockDec(o.lowMem) + dec.localFrame = newFrameDec(o) + dec.localFrame.o.dicts = nil d.decoders <- dec } + d.publishedOptions.Store(&o) if r == nil { return &d, nil @@ -171,6 +178,9 @@ d.drainOutput() d.syncStream.br.r = nil + if d.frame != nil { + d.frame.o.dicts = nil + } if r == nil { d.current.err = ErrDecoderNilInput if len(d.current.b) > 0 { @@ -180,8 +190,10 @@ return nil } + o := *d.publishedOptions.Load() + // If bytes buffer and < 5MB, do sync decoding anyway. - if bb, ok := r.(byter); ok && bb.Len() < d.o.decodeBufsBelow && !d.o.limitToCap { + if bb, ok := r.(byter); ok && bb.Len() < o.decodeBufsBelow && !o.limitToCap { bb2 := bb if debugDecoder { println("*bytes.Buffer detected, doing sync decode, len:", bb.Len()) @@ -193,18 +205,30 @@ } dst, err := d.DecodeAll(b, dst) - if err == nil { - err = io.EOF + if !errors.Is(err, ErrDecoderSizeExceeded) { + if err == nil { + err = io.EOF + } + // Save output buffer + d.syncStream.dstBuf = dst + d.current.b = dst + d.current.err = err + d.current.flushed = true + if debugDecoder { + println("sync decode to", len(dst), "bytes, err:", err) + } + return nil } - // Save output buffer - d.syncStream.dstBuf = dst - d.current.b = dst - d.current.err = err - d.current.flushed = true + // The output does not fit WithDecoderMaxMemory, which caps the total + // decoded size when decoding in memory. A Reader follows the + // streaming contract, where the option caps the window instead, so + // decode this input as a stream after all; the shortcut has not + // consumed the reader. Drop the oversized buffer rather than keep it + // for reuse. if debugDecoder { - println("sync decode to", len(dst), "bytes, err:", err) + println("sync decode exceeded max memory after", len(dst), "bytes; streaming instead") } - return nil + d.syncStream.dstBuf = nil } // Remove current block. d.stashDecoder() @@ -217,14 +241,16 @@ // Ensure no-one else is still running... d.streamWg.Wait() if d.frame == nil { - d.frame = newFrameDec(d.o) + d.frame = newFrameDec(o) + } else { + d.frame.setOptions(o) } - if d.o.concurrent == 1 { + if o.concurrent == 1 { return d.startSyncDecoder(r) } - d.current.output = make(chan decodeOutput, d.o.concurrent) + d.current.output = make(chan decodeOutput, o.concurrent) ctx, cancel := context.WithCancel(context.Background()) d.current.cancel = cancel d.streamWg.Add(1) @@ -238,13 +264,21 @@ // Options are applied on top of the existing options. // Some options cannot be changed on reset and will return an error. func (d *Decoder) ResetWithOptions(r io.Reader, opts ...DOption) error { - d.o.resetOpt = true - defer func() { d.o.resetOpt = false }() + if d.current.err == ErrDecoderClosed { + return d.current.err + } + next := *d.publishedOptions.Load() + next.dicts = maps.Clone(next.dicts) + next.resetOpt = true for _, o := range opts { - if err := o(&d.o); err != nil { + if err := o(&next); err != nil { + next.resetOpt = false + d.publishedOptions.Store(&next) return err } } + next.resetOpt = false + d.publishedOptions.Store(&next) return d.Reset(r) } @@ -324,6 +358,8 @@ // Grab a block decoder and frame decoder. block := <-d.decoders frame := block.localFrame + o := *d.publishedOptions.Load() + frame.setOptions(o) initialSize := len(dst) defer func() { if debugDecoder { @@ -335,6 +371,7 @@ frame.history.decoders.br.in = nil frame.history.decoders.br.cursor = 0 } + frame.o.dicts = nil d.decoders <- block }() frame.bBuf = input @@ -354,20 +391,20 @@ if err = d.setDict(frame); err != nil { return nil, err } - if frame.WindowSize > d.o.maxWindowSize { + if frame.WindowSize > frame.o.maxWindowSize { if debugDecoder { - println("window size exceeded:", frame.WindowSize, ">", d.o.maxWindowSize) + println("window size exceeded:", frame.WindowSize, ">", frame.o.maxWindowSize) } return dst, ErrWindowSizeExceeded } if frame.FrameContentSize != fcsUnknown { - if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)-initialSize) { + if frame.FrameContentSize > frame.o.maxDecodedSize-uint64(len(dst)-initialSize) { if debugDecoder { - println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", d.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst)) + println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", frame.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst)) } return dst, ErrDecoderSizeExceeded } - if d.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) { + if frame.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) { if debugDecoder { println("decoder size exceeded; fcs:", frame.FrameContentSize, "> (cap-len)", cap(dst)-len(dst)) } @@ -380,14 +417,14 @@ } } - if cap(dst) == 0 && !d.o.limitToCap { + if cap(dst) == 0 && !frame.o.limitToCap { // Allocate len(input) * 2 by default if nothing is provided // and we didn't get frame content size. size := min( // Cap to 1 MB. len(input)*2, 1<<20) - if uint64(size) > d.o.maxDecodedSize { - size = int(d.o.maxDecodedSize) + if uint64(size) > frame.o.maxDecodedSize { + size = int(frame.o.maxDecodedSize) } dst = make([]byte, 0, size) } @@ -396,7 +433,7 @@ if err != nil { return dst, err } - if uint64(len(dst)-initialSize) > d.o.maxDecodedSize { + if uint64(len(dst)-initialSize) > frame.o.maxDecodedSize { return dst, ErrDecoderSizeExceeded } if len(frame.bBuf) == 0 { @@ -459,7 +496,7 @@ println("got", len(d.current.b), "bytes, error:", d.current.err, "data crc:", tmp) } - if d.o.ignoreChecksum { + if d.frame.o.ignoreChecksum { return true } @@ -497,7 +534,7 @@ if d.current.err != nil { return false } - if d.frame.WindowSize > d.o.maxDecodedSize || d.frame.WindowSize > d.o.maxWindowSize { + if d.frame.WindowSize > d.frame.o.maxDecodedSize || d.frame.WindowSize > d.frame.o.maxWindowSize { d.current.err = ErrDecoderSizeExceeded return false } @@ -546,11 +583,11 @@ // Update/Check CRC if d.frame.HasCheckSum { - if !d.o.ignoreChecksum { + if !d.frame.o.ignoreChecksum { d.frame.crc.Write(d.current.b) } if d.current.d.Last { - if !d.o.ignoreChecksum { + if !d.frame.o.ignoreChecksum { d.current.err = d.frame.checkCRC() } else { d.current.err = d.frame.consumeCRC() @@ -656,8 +693,8 @@ defer d.streamWg.Done() br := readerWrapper{r: r} - var seqDecode = make(chan *blockDec, d.o.concurrent) - var seqExecute = make(chan *blockDec, d.o.concurrent) + var seqDecode = make(chan *blockDec, d.frame.o.concurrent) + var seqExecute = make(chan *blockDec, d.frame.o.concurrent) // Async 1: Decode sequences... go func() { @@ -858,9 +895,9 @@ if err == nil { err = d.setDict(frame) } - if err == nil && d.frame.WindowSize > d.o.maxWindowSize { + if err == nil && d.frame.WindowSize > d.frame.o.maxWindowSize { if debugDecoder { - println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.o.maxWindowSize) + println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.frame.o.maxWindowSize) } err = ErrDecoderSizeExceeded @@ -940,7 +977,7 @@ } func (d *Decoder) setDict(frame *frameDec) (err error) { - dict, ok := d.o.dicts[frame.DictionaryID] + dict, ok := frame.o.dicts[frame.DictionaryID] if ok { if debugDecoder { println("setting dict", frame.DictionaryID) --- a/vendor/github.com/klauspost/compress/zstd/dict.go +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -219,6 +219,9 @@ if len(contents) == 0 { return nil, errors.New("no content provided") } + if o.Level < speedNotSet || o.Level >= speedLast { + return nil, fmt.Errorf("unknown encoder level %d", o.Level) + } d := dict{ id: o.ID, litEnc: nil, @@ -296,40 +299,81 @@ if offset > 3 { newOffsets[offset-3]++ } else { - newOffsets[uint32(o.Offsets[offset-1])]++ + // Repeat codes reference the training Offsets. Skip unset + // (zero) entries so they are not ranked as real offsets. + prev := o.Offsets[offset-1] + if prev > 0 { + newOffsets[uint32(prev)]++ + } } } } // Find most used offsets. var sortedOffsets []uint32 for k := range newOffsets { + if k == 0 { + continue + } sortedOffsets = append(sortedOffsets, k) } sort.Slice(sortedOffsets, func(i, j int) bool { a, b := sortedOffsets[i], sortedOffsets[j] - if a == b { + ca, cb := newOffsets[a], newOffsets[b] + if ca == cb { // Prefer the longer offset - return sortedOffsets[i] > sortedOffsets[j] + return a > b } - return newOffsets[sortedOffsets[i]] > newOffsets[sortedOffsets[j]] + return ca > cb }) - if len(sortedOffsets) > 3 { - if debug { - print("Offsets:") - for i, v := range sortedOffsets { - if i > 20 { - break - } - printf("[%d: %d],", v, newOffsets[v]) + if debug { + print("Offsets:") + for i, v := range sortedOffsets { + if i > 20 { + break } - println("") + printf("[%d: %d],", v, newOffsets[v]) } - - sortedOffsets = sortedOffsets[:3] + println("") } - for i, v := range sortedOffsets { - o.Offsets[i] = int(v) + // Dictionary recent-offsets must be three positive values within the + // history. Ranked matches may be fewer (or empty when only unset + // repeat codes were seen), so fill remaining slots with defaults. + used := make(map[int]bool, 3) + var finalOffsets [3]int + nOff := 0 + for _, v := range sortedOffsets { + iv := int(v) + if iv <= 0 || iv > len(hist) || used[iv] { + continue + } + finalOffsets[nOff] = iv + used[iv] = true + nOff++ + if nOff == 3 { + break + } } + for _, def := range []int{1, 4, 8} { + if nOff == 3 { + break + } + if def <= len(hist) && !used[def] { + finalOffsets[nOff] = def + used[def] = true + nOff++ + } + } + for def := 1; nOff < 3 && def <= len(hist); def++ { + if !used[def] { + finalOffsets[nOff] = def + used[def] = true + nOff++ + } + } + if nOff < 3 { + return nil, fmt.Errorf("could not determine 3 valid dictionary offsets (history size %d)", len(hist)) + } + o.Offsets = finalOffsets if debug { println("New repeat offsets", o.Offsets) } @@ -337,6 +381,9 @@ if nUsed == 0 || seqs == 0 { return nil, fmt.Errorf("%d blocks, %d sequences found", nUsed, seqs) } + if litTotal == 0 { + return nil, errors.New("0 literals found") + } if debug { println("Sequences:", seqs, "Blocks:", nUsed, "Literals:", litTotal) } @@ -517,11 +564,10 @@ out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[1]))) out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[2]))) out.Write(hist) + if _, err := loadDict(out.Bytes()); err != nil { + return nil, fmt.Errorf("built dictionary failed validation: %w", err) + } if debug { - _, err := loadDict(out.Bytes()) - if err != nil { - panic(err) - } i, err := InspectDictionary(out.Bytes()) if err != nil { panic(err) --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -49,13 +49,16 @@ ) func newFrameDec(o decoderOptions) *frameDec { + d := frameDec{} + d.setOptions(o) + return &d +} + +func (d *frameDec) setOptions(o decoderOptions) { if o.maxWindowSize > o.maxDecodedSize { o.maxWindowSize = o.maxDecodedSize } - d := frameDec{ - o: o, - } - return &d + d.o = o } // reset will read the frame header and prepare for block decoding. --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s @@ -1,7 +1,7 @@ // Code generated by command: go run gen_fse.go -out ../fse_decoder.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. // EXPERIMENTAL arm64 output lowered from an amd64 avo program. -//go:build arm64 && !appengine && !noasm && gc && !noasm +//go:build arm64 && (!appengine && !noasm && gc && !noasm) // func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int TEXT ·buildDtable_asm(SB), $0-24 @@ -26,9 +26,11 @@ JMP init_main_loop_condition init_main_loop: - ADD R8<<1, R1, R15 - MOVH (R15), R9 - CMN $1, R9 + MOVH (R1)(R8<<1), R9 + MOVD $-1, R16 + AND $0xffff, R16, R16 + AND $0xffff, R9, R15 + CMP R16, R15 BNE do_not_update_high_threshold ADD R7<<3, R5, R15 MOVB R8, 1(R15) @@ -36,8 +38,7 @@ MOVD $0x0000000000000001, R9 do_not_update_high_threshold: - ADD R8<<1, R3, R15 - MOVH R9, (R15) + MOVH R9, (R3)(R8<<1) ADD $1, R8, R8 init_main_loop_condition: @@ -46,12 +47,10 @@ // Spread symbols // Calculate table step - MOVD R0, R8 - LSR $0x01, R8, R8 - MOVD R0, R9 - LSR $0x03, R9, R9 - ADD R9, R8, R8 - ADD $3, R8, R8 + LSR $0x01, R0, R8 + LSR $0x03, R0, R9 + ADD R9, R8, R8 + ADD $3, R8, R8 // Fill add bits values SUB $1, R0, R9 @@ -61,8 +60,7 @@ spread_main_loop: MOVD $0, R12 - ADD R11<<1, R1, R15 - MOVH (R15), R13 + MOVH (R1)(R11<<1), R13 JMP spread_inner_loop_condition spread_inner_loop: @@ -99,11 +97,9 @@ build_table_main_table: ADD R6<<3, R5, R15 MOVBU 1(R15), R1 - ADD R1<<1, R3, R15 - MOVHU (R15), R7 + MOVHU (R3)(R1<<1), R7 ADD $1, R7, R8 - ADD R1<<1, R3, R15 - MOVH R8, (R15) + MOVH R8, (R3)(R1<<1) MOVD R7, R8 CLZ R8, R16 MOVD $63, R8 @@ -126,9 +122,11 @@ RET build_table_check1_ok: - TST R1, R1 + TST $0xff, R1 BNE build_table_check2_ok - CMP R6, R7 + AND $0xffff, R6, R16 + AND $0xffff, R7, R15 + CMP R16, R15 BNE build_table_check2_ok MOVD ctx+8(FP), R0 MOVD R7, 24(R0) --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go @@ -52,6 +52,11 @@ d.n = 0 } +// maxAsmSize bounds the input handed to a single writeBlocks call. Assembly is +// never preemptible, so an unbounded call holds every P in stop-the-world for +// its duration. Whole 32-byte blocks, so Write can feed it in pieces. +const maxAsmSize = 128 << 10 // 4096 whole 32-byte blocks + // Size always returns 8 bytes. func (d *Digest) Size() int { return 8 } @@ -83,6 +88,13 @@ d.n = 0 } + for len(b) >= maxAsmSize { + // Assembly is not preemptible, so hashing a large buffer in one call + // blocks every stop-the-world for the whole call. Feed it in chunks. + writeBlocks(d, b[:maxAsmSize]) + b = b[maxAsmSize:] + } + if len(b) >= 32 { // One or more full blocks left. nw := writeBlocks(d, b) --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s @@ -175,7 +175,11 @@ RET // func writeBlocks(d *Digest, b []byte) int -TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 +// +// Deliberately not NOSPLIT: the stack check is the preemption point that lets a +// stop-the-world proceed between chunks. The frame must be >= abi.StackSmall +// (128), or the assembler marks this leaf NOSPLIT anyway and drops that check. +TEXT ·writeBlocks(SB), $128-40 // Load fixed primes needed for round. MOVQ ·primes+0(SB), prime1 MOVQ ·primes+8(SB), prime2 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s @@ -163,7 +163,11 @@ RET // func writeBlocks(s *Digest, b []byte) int -TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 +// +// Deliberately not NOSPLIT: the stack check is the preemption point that lets a +// stop-the-world proceed between chunks. The frame must be >= abi.StackSmall +// (128), or the assembler marks this leaf NOSPLIT anyway and drops that check. +TEXT ·writeBlocks(SB), $128-40 LDP ·primes+0(SB), (prime1, prime2) // Load state. Assume v[1-4] are stored contiguously. --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -99,6 +99,29 @@ return nil } +// consumeSyncLen accounts for n bytes of frame output produced outside +// decodeSync. maxSyncLen bounds how much the frame may still produce and +// decodeSyncSimple compares it with the output buffer's slack to decide +// whether the extended 16-byte copies are safe. Blocks with sequences +// subtracted their output; raw blocks, RLE blocks and compressed blocks +// holding only literals did not, so a frame that opened with a raw block +// overstated its remaining size for every block after it and ran the +// bounds-exact copies for the rest of the frame. +// +// Zero means "no bound" and selects the conservative paths, so a block that +// meets or exceeds the bound lands there; the frame-size checks in +// runDecoder reject the excess afterwards. +func (s *sequenceDecs) consumeSyncLen(n int) { + if s.maxSyncLen == 0 { + return + } + if uint64(n) >= s.maxSyncLen { + s.maxSyncLen = 0 + return + } + s.maxSyncLen -= uint64(n) +} + func (s *sequenceDecs) freeDecoders() { if f := s.litLengths.fse; f != nil && !f.preDefined { fseDecoderPool.Put(f) --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -4,7 +4,7 @@ // func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV -TEXT ·sequenceDecs_decode_amd64(SB), $8-32 +TEXT ·sequenceDecs_decode_amd64(SB), $40-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -16,6 +16,14 @@ MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 + MOVQ (AX), CX + MOVQ CX, 8(SP) + MOVQ 24(AX), CX + MOVQ CX, 16(SP) + MOVQ 48(AX), CX + MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) MOVQ 104(AX), R10 MOVQ s+0(FP), AX MOVQ 144(AX), R11 @@ -55,46 +63,32 @@ sequenceDecs_decode_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_of_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_amd64_of_update_zero: - MOVQ AX, 16(R10) + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, 16(R10) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_amd64_ml_update_zero: - MOVQ AX, 8(R10) + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, 8(R10) // Fill bitreader to have enough for the remaining CMPQ SI, $0x08 @@ -126,90 +120,76 @@ sequenceDecs_decode_amd64_fill_2_end: // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_amd64_ll_update_zero: - MOVQ AX, (R10) + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, (R10) // Fill bitreader for state updates MOVQ R14, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_amd64_skip_update // Update Literal Length State MOVBQZX DI, R14 SHRL $0x10, DI - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R14 SHRL $0x10, R8 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R14 SHRL $0x10, R9 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decode_amd64_skip_update: @@ -290,8 +270,7 @@ sequenceDecs_decode_amd64_match_len_ofs_ok: ADDQ $0x18, R10 - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decode_amd64_main_loop MOVQ s+0(FP), AX MOVQ R11, 144(AX) @@ -303,36 +282,54 @@ MOVQ SI, 32(AX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_amd64_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_amd64_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV -TEXT ·sequenceDecs_decode_56_amd64(SB), $8-32 +TEXT ·sequenceDecs_decode_56_amd64(SB), $40-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -344,6 +341,14 @@ MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 + MOVQ (AX), CX + MOVQ CX, 8(SP) + MOVQ 24(AX), CX + MOVQ CX, 16(SP) + MOVQ 48(AX), CX + MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) MOVQ 104(AX), R10 MOVQ s+0(FP), AX MOVQ 144(AX), R11 @@ -383,132 +388,104 @@ sequenceDecs_decode_56_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_of_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_56_amd64_of_update_zero: - MOVQ AX, 16(R10) + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, 16(R10) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_56_amd64_ml_update_zero: - MOVQ AX, 8(R10) + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, 8(R10) // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_56_amd64_ll_update_zero: - MOVQ AX, (R10) + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, (R10) // Fill bitreader for state updates MOVQ R14, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_56_amd64_skip_update // Update Literal Length State MOVBQZX DI, R14 SHRL $0x10, DI - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R14 SHRL $0x10, R8 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R14 SHRL $0x10, R9 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decode_56_amd64_skip_update: @@ -589,8 +566,7 @@ sequenceDecs_decode_56_amd64_match_len_ofs_ok: ADDQ $0x18, R10 - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decode_56_amd64_main_loop MOVQ s+0(FP), AX MOVQ R11, 144(AX) @@ -602,36 +578,54 @@ MOVQ SI, 32(AX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_56_amd64_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_56_amd64_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: BMI, BMI2, CMOV -TEXT ·sequenceDecs_decode_bmi2(SB), $8-32 +TEXT ·sequenceDecs_decode_bmi2(SB), $40-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -643,6 +637,14 @@ MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 + MOVQ (CX), R9 + MOVQ R9, 8(SP) + MOVQ 24(CX), R9 + MOVQ R9, 16(SP) + MOVQ 48(CX), R9 + MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) MOVQ 104(CX), R9 MOVQ s+0(FP), CX MOVQ 144(CX), R10 @@ -753,8 +755,7 @@ MOVQ R13, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R13 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_bmi2_skip_update LEAQ (SI)(DI*1), R14 ADDQ R8, R14 @@ -772,8 +773,7 @@ ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -783,8 +783,7 @@ ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -793,8 +792,7 @@ ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decode_bmi2_skip_update: @@ -875,8 +873,7 @@ sequenceDecs_decode_bmi2_match_len_ofs_ok: ADDQ $0x18, R9 - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decode_bmi2_main_loop MOVQ s+0(FP), CX MOVQ R10, 144(CX) @@ -888,36 +885,54 @@ MOVQ BX, 32(CX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_bmi2_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_bmi2_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: BMI, BMI2, CMOV -TEXT ·sequenceDecs_decode_56_bmi2(SB), $8-32 +TEXT ·sequenceDecs_decode_56_bmi2(SB), $40-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -929,6 +944,14 @@ MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 + MOVQ (CX), R9 + MOVQ R9, 8(SP) + MOVQ 24(CX), R9 + MOVQ R9, 16(SP) + MOVQ 48(CX), R9 + MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) MOVQ 104(CX), R9 MOVQ s+0(FP), CX MOVQ 144(CX), R10 @@ -1010,8 +1033,7 @@ MOVQ R13, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R13 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_56_bmi2_skip_update LEAQ (SI)(DI*1), R14 ADDQ R8, R14 @@ -1029,8 +1051,7 @@ ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -1040,8 +1061,7 @@ ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -1050,8 +1070,7 @@ ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decode_56_bmi2_skip_update: @@ -1132,8 +1151,7 @@ sequenceDecs_decode_56_bmi2_match_len_ofs_ok: ADDQ $0x18, R9 - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decode_56_bmi2_main_loop MOVQ s+0(FP), CX MOVQ R10, 144(CX) @@ -1145,30 +1163,48 @@ MOVQ BX, 32(CX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_56_bmi2_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_56_bmi2_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET @@ -1203,9 +1239,11 @@ MOVQ 8(AX), R13 // Copy literals - TESTQ R11, R11 - JZ check_offset - XORQ R14, R14 + MOVUPS (SI), X0 + MOVUPS X0, (BX) + CMPQ R11, $0x10 + JBE copy_1_end + MOVQ $0x00000010, R14 copy_1: MOVUPS (SI)(R14*1), X0 @@ -1213,12 +1251,13 @@ ADDQ $0x10, R14 CMPQ R14, R11 JB copy_1 - ADDQ R11, SI - ADDQ R11, BX - ADDQ R11, DI + +copy_1_end: + ADDQ R11, SI + ADDQ R11, BX + ADDQ R11, DI // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: LEAQ (DI)(R10*1), R11 CMPQ R12, R11 JG error_match_off_too_big @@ -1258,13 +1297,13 @@ JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), R11 - MOVB 2(R14), R12 - MOVW R11, (BX) - MOVB R12, 2(BX) - ADDQ R13, R14 - ADDQ R13, BX - JMP copy_4_end + MOVWQZX (R14), R11 + MOVB 2(R14), R12 + MOVW R11, (BX) + MOVB R12, 2(BX) + ADDQ R13, R14 + ADDQ R13, BX + JMP copy_4_end copy_4_move_4through7: MOVL (R14), R11 @@ -1327,13 +1366,13 @@ JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (BX) - MOVB BP, 2(BX) - ADDQ R11, R14 - ADDQ R11, BX - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (BX) + MOVB BP, 2(BX) + ADDQ R11, R14 + ADDQ R11, BX + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 @@ -1366,9 +1405,19 @@ JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, DI - MOVQ BX, R12 + ADDQ R13, DI + MOVUPS (R11), X0 + MOVUPS X0, (BX) + CMPQ R13, $0x10 + JA copy_2_long + ADDQ R13, BX + JMP handle_loop + +copy_2_long: + LEAQ 16(BX), R12 ADDQ R13, BX + ADDQ $0x10, R11 + SUBQ $0x10, R13 copy_2: MOVUPS (R11), X0 @@ -1384,12 +1433,12 @@ ADDQ R13, DI copy_slow_3: - MOVB (R11), R12 - MOVB R12, (BX) - INCQ R11 - INCQ BX - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (R11), R12 + MOVB R12, (BX) + INCQ R11 + INCQ BX + DECQ R13 + JNZ copy_slow_3 handle_loop: ADDQ $0x18, AX @@ -1494,13 +1543,13 @@ JMP copy_1_end copy_1_move_3: - MOVW (SI), R14 - MOVB 2(SI), R15 - MOVW R14, (BX) - MOVB R15, 2(BX) - ADDQ R11, SI - ADDQ R11, BX - JMP copy_1_end + MOVWQZX (SI), R14 + MOVB 2(SI), R15 + MOVW R14, (BX) + MOVB R15, 2(BX) + ADDQ R11, SI + ADDQ R11, BX + JMP copy_1_end copy_1_move_4through7: MOVL (SI), R14 @@ -1563,13 +1612,13 @@ JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), R11 - MOVB 2(R14), R12 - MOVW R11, (BX) - MOVB R12, 2(BX) - ADDQ R13, R14 - ADDQ R13, BX - JMP copy_4_end + MOVWQZX (R14), R11 + MOVB 2(R14), R12 + MOVW R11, (BX) + MOVB R12, 2(BX) + ADDQ R13, R14 + ADDQ R13, BX + JMP copy_4_end copy_4_move_4through7: MOVL (R14), R11 @@ -1632,13 +1681,13 @@ JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (BX) - MOVB BP, 2(BX) - ADDQ R11, R14 - ADDQ R11, BX - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (BX) + MOVB BP, 2(BX) + ADDQ R11, R14 + ADDQ R11, BX + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 @@ -1707,13 +1756,13 @@ JMP copy_2_end copy_2_move_3: - MOVW (R11), R12 - MOVB 2(R11), R14 - MOVW R12, (BX) - MOVB R14, 2(BX) - ADDQ R13, R11 - ADDQ R13, BX - JMP copy_2_end + MOVWQZX (R11), R12 + MOVB 2(R11), R14 + MOVW R12, (BX) + MOVB R14, 2(BX) + ADDQ R13, R11 + ADDQ R13, BX + JMP copy_2_end copy_2_move_4through7: MOVL (R11), R12 @@ -1740,12 +1789,12 @@ ADDQ R13, DI copy_slow_3: - MOVB (R11), R12 - MOVB R12, (BX) - INCQ R11 - INCQ BX - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (R11), R12 + MOVB R12, (BX) + INCQ R11 + INCQ BX + DECQ R13 + JNZ copy_slow_3 handle_loop: ADDQ $0x18, AX @@ -1784,7 +1833,7 @@ // func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_amd64(SB), $96-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -1796,26 +1845,34 @@ MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 - XORQ CX, CX + MOVQ (AX), CX MOVQ CX, 8(SP) + MOVQ 24(AX), CX MOVQ CX, 16(SP) + MOVQ 48(AX), CX MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) + XORQ CX, CX + MOVQ CX, 40(SP) + MOVQ CX, 48(SP) + MOVQ CX, 56(SP) MOVQ 112(AX), R10 MOVQ 128(AX), CX - MOVQ CX, 32(SP) + MOVQ CX, 64(SP) MOVQ 144(AX), R11 MOVQ 136(AX), R12 MOVQ 200(AX), CX - MOVQ CX, 56(SP) + MOVQ CX, 88(SP) MOVQ 176(AX), CX - MOVQ CX, 48(SP) + MOVQ CX, 80(SP) MOVQ 184(AX), AX - MOVQ AX, 40(SP) - MOVQ 40(SP), AX - ADDQ AX, 48(SP) + MOVQ AX, 72(SP) + MOVQ 72(SP), AX + ADDQ AX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R10, 32(SP) + ADDQ R10, 64(SP) // outBase += outPosition ADDQ R12, R10 @@ -1853,46 +1910,32 @@ sequenceDecs_decodeSync_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_of_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_of_update_zero: - MOVQ AX, 8(SP) + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 40(SP) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_ml_update_zero: - MOVQ AX, 16(SP) + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ SI, $0x08 @@ -1924,96 +1967,82 @@ sequenceDecs_decodeSync_amd64_fill_2_end: // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_ll_update_zero: - MOVQ AX, 24(SP) + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 56(SP) // Fill bitreader for state updates MOVQ R13, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_amd64_skip_update // Update Literal Length State MOVBQZX DI, R13 SHRL $0x10, DI - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R13 SHRL $0x10, R8 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R13 SHRL $0x10, R9 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decodeSync_amd64_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ AX, $0x01 JBE sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -2022,7 +2051,7 @@ JMP sequenceDecs_decodeSync_amd64_after_adjust sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_amd64_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_amd64_adjust_offset_nonzero @@ -2057,14 +2086,11 @@ MOVQ R14, R13 sequenceDecs_decodeSync_amd64_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), AX - MOVQ 24(SP), CX - LEAQ (AX)(CX*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), AX + MOVQ 56(SP), CX MOVQ ctx+16(FP), R14 SUBQ CX, 104(R14) JS error_not_enough_literals @@ -2076,50 +2102,49 @@ JNZ sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_amd64_match_len_ofs_ok: - MOVQ 24(SP), AX - MOVQ 8(SP), CX - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ (AX)(R13*1), R14 + LEAQ 16(CX)(AX*1), R14 ADDQ R10, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ AX, AX - JZ check_offset - XORQ R14, R14 + MOVUPS (R11), X0 + MOVUPS X0, (R10) + CMPQ CX, $0x10 + JBE copy_1_end + MOVQ $0x00000010, R14 copy_1: MOVUPS (R11)(R14*1), X0 MOVUPS X0, (R10)(R14*1) ADDQ $0x10, R14 - CMPQ R14, AX + CMPQ R14, CX JB copy_1 - ADDQ AX, R11 - ADDQ AX, R10 - ADDQ AX, R12 + +copy_1_end: + ADDQ CX, R11 + ADDQ CX, R10 + ADDQ CX, R12 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVQ R12, AX - ADDQ 40(SP), AX - CMPQ CX, AX + MOVQ R12, CX + ADDQ 72(SP), CX + CMPQ R13, CX JG error_match_off_too_big - CMPQ CX, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ CX, AX - SUBQ R12, AX + MOVQ R13, CX + SUBQ R12, CX JLS copy_match - MOVQ 48(SP), R14 - SUBQ AX, R14 - CMPQ R13, AX + MOVQ 80(SP), R14 + SUBQ CX, R14 + CMPQ AX, CX JG copy_all_from_history - MOVQ R13, AX - SUBQ $0x10, AX + MOVQ AX, CX + SUBQ $0x10, CX JB copy_4_small copy_4_loop: @@ -2127,54 +2152,54 @@ MOVUPS X0, (R10) ADDQ $0x10, R14 ADDQ $0x10, R10 - SUBQ $0x10, AX + SUBQ $0x10, CX JAE copy_4_loop - LEAQ 16(R14)(AX*1), R14 - LEAQ 16(R10)(AX*1), R10 + LEAQ 16(R14)(CX*1), R14 + LEAQ 16(R10)(CX*1), R10 MOVUPS -16(R14), X0 MOVUPS X0, -16(R10) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ AX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ AX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), AX - MOVB 2(R14), CL - MOVW AX, (R10) - MOVB CL, 2(R10) - ADDQ R13, R14 - ADDQ R13, R10 - JMP copy_4_end + MOVWQZX (R14), CX + MOVB 2(R14), R13 + MOVW CX, (R10) + MOVB R13, 2(R10) + ADDQ AX, R14 + ADDQ AX, R10 + JMP copy_4_end copy_4_move_4through7: - MOVL (R14), AX - MOVL -4(R14)(R13*1), CX - MOVL AX, (R10) - MOVL CX, -4(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVL (R14), CX + MOVL -4(R14)(AX*1), R13 + MOVL CX, (R10) + MOVL R13, -4(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), AX - MOVQ -8(R14)(R13*1), CX - MOVQ AX, (R10) - MOVQ CX, -8(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVQ (R14), CX + MOVQ -8(R14)(AX*1), R13 + MOVQ CX, (R10) + MOVQ R13, -8(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 copy_4_end: - ADDQ R13, R12 + ADDQ AX, R12 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ AX, R15 + MOVQ CX, R15 SUBQ $0x10, R15 JB copy_5_small @@ -2192,90 +2217,99 @@ JMP copy_5_end copy_5_small: - CMPQ AX, $0x03 + CMPQ CX, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ AX, $0x08 + CMPQ CX, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(AX*1), BP + MOVB -1(R14)(CX*1), BP MOVB R15, (R10) - MOVB BP, -1(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVB BP, -1(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R10) - MOVB BP, 2(R10) - ADDQ AX, R14 - ADDQ AX, R10 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R10) + MOVB BP, 2(R10) + ADDQ CX, R14 + ADDQ CX, R10 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(AX*1), BP + MOVL -4(R14)(CX*1), BP MOVL R15, (R10) - MOVL BP, -4(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVL BP, -4(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(AX*1), BP + MOVQ -8(R14)(CX*1), BP MOVQ R15, (R10) - MOVQ BP, -8(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVQ BP, -8(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 copy_5_end: - ADDQ AX, R12 - SUBQ AX, R13 + ADDQ CX, R12 + SUBQ CX, AX // Copy match from the current buffer copy_match: - MOVQ R10, AX - SUBQ CX, AX + MOVQ R10, CX + SUBQ R13, CX // ml <= mo - CMPQ R13, CX + CMPQ AX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R12 - MOVQ R10, CX - ADDQ R13, R10 + ADDQ AX, R12 + MOVUPS (CX), X0 + MOVUPS X0, (R10) + CMPQ AX, $0x10 + JA copy_2_long + ADDQ AX, R10 + JMP handle_loop + +copy_2_long: + LEAQ 16(R10), R13 + ADDQ AX, R10 + ADDQ $0x10, CX + SUBQ $0x10, AX copy_2: - MOVUPS (AX), X0 - MOVUPS X0, (CX) - ADDQ $0x10, AX + MOVUPS (CX), X0 + MOVUPS X0, (R13) ADDQ $0x10, CX - SUBQ $0x10, R13 + ADDQ $0x10, R13 + SUBQ $0x10, AX JHI copy_2 JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R12 + ADDQ AX, R12 copy_slow_3: - MOVB (AX), CL - MOVB CL, (R10) - INCQ AX - INCQ R10 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (CX), R13 + MOVB R13, (R10) + INCQ CX + INCQ R10 + DECQ AX + JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_amd64_main_loop loop_finished: @@ -2284,6 +2318,13 @@ MOVB BL, 40(AX) MOVQ SI, 32(AX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R12, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R12, 136(AX) @@ -2297,7 +2338,7 @@ // Return with match length error sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -2306,7 +2347,7 @@ // Return with match too long error sequenceDecs_decodeSync_amd64_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -2314,7 +2355,7 @@ // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R12, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -2323,7 +2364,7 @@ // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -2336,9 +2377,9 @@ // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R12, 136(AX) MOVQ $0x00000005, ret+24(FP) @@ -2346,7 +2387,7 @@ // func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: BMI, BMI2, CMOV, SSE -TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_bmi2(SB), $96-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -2358,26 +2399,34 @@ MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 - XORQ R9, R9 + MOVQ (CX), R9 MOVQ R9, 8(SP) + MOVQ 24(CX), R9 MOVQ R9, 16(SP) + MOVQ 48(CX), R9 MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) + XORQ R9, R9 + MOVQ R9, 40(SP) + MOVQ R9, 48(SP) + MOVQ R9, 56(SP) MOVQ 112(CX), R9 MOVQ 128(CX), R10 - MOVQ R10, 32(SP) + MOVQ R10, 64(SP) MOVQ 144(CX), R10 MOVQ 136(CX), R11 MOVQ 200(CX), R12 - MOVQ R12, 56(SP) + MOVQ R12, 88(SP) MOVQ 176(CX), R12 - MOVQ R12, 48(SP) + MOVQ R12, 80(SP) MOVQ 184(CX), CX - MOVQ CX, 40(SP) - MOVQ 40(SP), CX - ADDQ CX, 48(SP) + MOVQ CX, 72(SP) + MOVQ 72(SP), CX + ADDQ CX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R9, 32(SP) + ADDQ R9, 64(SP) // outBase += outPosition ADDQ R11, R9 @@ -2425,7 +2474,7 @@ MOVQ R8, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 8(SP) + MOVQ CX, 40(SP) // Update match length MOVQ $0x00000808, CX @@ -2438,7 +2487,7 @@ MOVQ DI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 16(SP) + MOVQ CX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ BX, $0x08 @@ -2480,14 +2529,13 @@ MOVQ SI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 24(SP) + MOVQ CX, 56(SP) // Fill bitreader for state updates MOVQ R12, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R12 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_bmi2_skip_update LEAQ (SI)(DI*1), R13 ADDQ R8, R13 @@ -2505,8 +2553,7 @@ ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -2516,8 +2563,7 @@ ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -2526,14 +2572,13 @@ ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decodeSync_bmi2_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ R12, $0x01 JBE sequenceDecs_decodeSync_bmi2_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -2542,7 +2587,7 @@ JMP sequenceDecs_decodeSync_bmi2_after_adjust sequenceDecs_decodeSync_bmi2_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_bmi2_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_bmi2_adjust_offset_nonzero @@ -2577,14 +2622,11 @@ MOVQ R14, R13 sequenceDecs_decodeSync_bmi2_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), CX - MOVQ 24(SP), R12 - LEAQ (CX)(R12*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), CX + MOVQ 56(SP), R12 MOVQ ctx+16(FP), R14 SUBQ R12, 104(R14) JS error_not_enough_literals @@ -2596,50 +2638,49 @@ JNZ sequenceDecs_decodeSync_bmi2_error_match_len_ofs_mismatch sequenceDecs_decodeSync_bmi2_match_len_ofs_ok: - MOVQ 24(SP), CX - MOVQ 8(SP), R12 - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ (CX)(R13*1), R14 + LEAQ 16(R12)(CX*1), R14 ADDQ R9, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ CX, CX - JZ check_offset - XORQ R14, R14 + MOVUPS (R10), X0 + MOVUPS X0, (R9) + CMPQ R12, $0x10 + JBE copy_1_end + MOVQ $0x00000010, R14 copy_1: MOVUPS (R10)(R14*1), X0 MOVUPS X0, (R9)(R14*1) ADDQ $0x10, R14 - CMPQ R14, CX + CMPQ R14, R12 JB copy_1 - ADDQ CX, R10 - ADDQ CX, R9 - ADDQ CX, R11 + +copy_1_end: + ADDQ R12, R10 + ADDQ R12, R9 + ADDQ R12, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVQ R11, CX - ADDQ 40(SP), CX - CMPQ R12, CX + MOVQ R11, R12 + ADDQ 72(SP), R12 + CMPQ R13, R12 JG error_match_off_too_big - CMPQ R12, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ R12, CX - SUBQ R11, CX + MOVQ R13, R12 + SUBQ R11, R12 JLS copy_match - MOVQ 48(SP), R14 - SUBQ CX, R14 - CMPQ R13, CX + MOVQ 80(SP), R14 + SUBQ R12, R14 + CMPQ CX, R12 JG copy_all_from_history - MOVQ R13, CX - SUBQ $0x10, CX + MOVQ CX, R12 + SUBQ $0x10, R12 JB copy_4_small copy_4_loop: @@ -2647,54 +2688,54 @@ MOVUPS X0, (R9) ADDQ $0x10, R14 ADDQ $0x10, R9 - SUBQ $0x10, CX + SUBQ $0x10, R12 JAE copy_4_loop - LEAQ 16(R14)(CX*1), R14 - LEAQ 16(R9)(CX*1), R9 + LEAQ 16(R14)(R12*1), R14 + LEAQ 16(R9)(R12*1), R9 MOVUPS -16(R14), X0 MOVUPS X0, -16(R9) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ CX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ CX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), CX - MOVB 2(R14), R12 - MOVW CX, (R9) - MOVB R12, 2(R9) - ADDQ R13, R14 - ADDQ R13, R9 - JMP copy_4_end + MOVWQZX (R14), R12 + MOVB 2(R14), R13 + MOVW R12, (R9) + MOVB R13, 2(R9) + ADDQ CX, R14 + ADDQ CX, R9 + JMP copy_4_end copy_4_move_4through7: - MOVL (R14), CX - MOVL -4(R14)(R13*1), R12 - MOVL CX, (R9) - MOVL R12, -4(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVL (R14), R12 + MOVL -4(R14)(CX*1), R13 + MOVL R12, (R9) + MOVL R13, -4(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), CX - MOVQ -8(R14)(R13*1), R12 - MOVQ CX, (R9) - MOVQ R12, -8(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVQ (R14), R12 + MOVQ -8(R14)(CX*1), R13 + MOVQ R12, (R9) + MOVQ R13, -8(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 copy_4_end: - ADDQ R13, R11 + ADDQ CX, R11 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ CX, R15 + MOVQ R12, R15 SUBQ $0x10, R15 JB copy_5_small @@ -2712,90 +2753,99 @@ JMP copy_5_end copy_5_small: - CMPQ CX, $0x03 + CMPQ R12, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ CX, $0x08 + CMPQ R12, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(CX*1), BP + MOVB -1(R14)(R12*1), BP MOVB R15, (R9) - MOVB BP, -1(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVB BP, -1(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R9) - MOVB BP, 2(R9) - ADDQ CX, R14 - ADDQ CX, R9 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R9) + MOVB BP, 2(R9) + ADDQ R12, R14 + ADDQ R12, R9 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(CX*1), BP + MOVL -4(R14)(R12*1), BP MOVL R15, (R9) - MOVL BP, -4(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVL BP, -4(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(CX*1), BP + MOVQ -8(R14)(R12*1), BP MOVQ R15, (R9) - MOVQ BP, -8(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVQ BP, -8(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 copy_5_end: - ADDQ CX, R11 - SUBQ CX, R13 + ADDQ R12, R11 + SUBQ R12, CX // Copy match from the current buffer copy_match: - MOVQ R9, CX - SUBQ R12, CX + MOVQ R9, R12 + SUBQ R13, R12 // ml <= mo - CMPQ R13, R12 + CMPQ CX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R11 - MOVQ R9, R12 - ADDQ R13, R9 + ADDQ CX, R11 + MOVUPS (R12), X0 + MOVUPS X0, (R9) + CMPQ CX, $0x10 + JA copy_2_long + ADDQ CX, R9 + JMP handle_loop + +copy_2_long: + LEAQ 16(R9), R13 + ADDQ CX, R9 + ADDQ $0x10, R12 + SUBQ $0x10, CX copy_2: - MOVUPS (CX), X0 - MOVUPS X0, (R12) - ADDQ $0x10, CX + MOVUPS (R12), X0 + MOVUPS X0, (R13) ADDQ $0x10, R12 - SUBQ $0x10, R13 + ADDQ $0x10, R13 + SUBQ $0x10, CX JHI copy_2 JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R11 + ADDQ CX, R11 copy_slow_3: - MOVB (CX), R12 - MOVB R12, (R9) - INCQ CX - INCQ R9 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (R12), R13 + MOVB R13, (R9) + INCQ R12 + INCQ R9 + DECQ CX + JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_bmi2_main_loop loop_finished: @@ -2804,6 +2854,13 @@ MOVB DL, 40(CX) MOVQ BX, 32(CX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R11, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R11, 136(AX) @@ -2817,7 +2874,7 @@ // Return with match length error sequenceDecs_decodeSync_bmi2_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -2826,7 +2883,7 @@ // Return with match too long error sequenceDecs_decodeSync_bmi2_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -2834,7 +2891,7 @@ // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R11, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -2843,7 +2900,7 @@ // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -2856,9 +2913,9 @@ // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R11, 136(AX) MOVQ $0x00000005, ret+24(FP) @@ -2866,7 +2923,7 @@ // func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $96-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -2878,26 +2935,34 @@ MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 - XORQ CX, CX + MOVQ (AX), CX MOVQ CX, 8(SP) + MOVQ 24(AX), CX MOVQ CX, 16(SP) + MOVQ 48(AX), CX MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) + XORQ CX, CX + MOVQ CX, 40(SP) + MOVQ CX, 48(SP) + MOVQ CX, 56(SP) MOVQ 112(AX), R10 MOVQ 128(AX), CX - MOVQ CX, 32(SP) + MOVQ CX, 64(SP) MOVQ 144(AX), R11 MOVQ 136(AX), R12 MOVQ 200(AX), CX - MOVQ CX, 56(SP) + MOVQ CX, 88(SP) MOVQ 176(AX), CX - MOVQ CX, 48(SP) + MOVQ CX, 80(SP) MOVQ 184(AX), AX - MOVQ AX, 40(SP) - MOVQ 40(SP), AX - ADDQ AX, 48(SP) + MOVQ AX, 72(SP) + MOVQ 72(SP), AX + ADDQ AX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R10, 32(SP) + ADDQ R10, 64(SP) // outBase += outPosition ADDQ R12, R10 @@ -2935,46 +3000,32 @@ sequenceDecs_decodeSync_safe_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_of_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_of_update_zero: - MOVQ AX, 8(SP) + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 40(SP) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_ml_update_zero: - MOVQ AX, 16(SP) + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ SI, $0x08 @@ -3006,96 +3057,82 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_end: // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_ll_update_zero: - MOVQ AX, 24(SP) + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 56(SP) // Fill bitreader for state updates MOVQ R13, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_safe_amd64_skip_update // Update Literal Length State MOVBQZX DI, R13 SHRL $0x10, DI - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R13 SHRL $0x10, R8 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R13 SHRL $0x10, R9 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decodeSync_safe_amd64_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ AX, $0x01 JBE sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -3104,7 +3141,7 @@ JMP sequenceDecs_decodeSync_safe_amd64_after_adjust sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero @@ -3139,14 +3176,11 @@ MOVQ R14, R13 sequenceDecs_decodeSync_safe_amd64_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), AX - MOVQ 24(SP), CX - LEAQ (AX)(CX*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), AX + MOVQ 56(SP), CX MOVQ ctx+16(FP), R14 SUBQ CX, 104(R14) JS error_not_enough_literals @@ -3158,20 +3192,16 @@ JNZ sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok: - MOVQ 24(SP), AX - MOVQ 8(SP), CX - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ (AX)(R13*1), R14 + LEAQ (CX)(AX*1), R14 ADDQ R10, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ AX, AX + TESTQ CX, CX JZ check_offset - MOVQ AX, R14 + MOVQ CX, R14 SUBQ $0x10, R14 JB copy_1_small @@ -3189,70 +3219,70 @@ JMP copy_1_end copy_1_small: - CMPQ AX, $0x03 + CMPQ CX, $0x03 JE copy_1_move_3 JB copy_1_move_1or2 - CMPQ AX, $0x08 + CMPQ CX, $0x08 JB copy_1_move_4through7 JMP copy_1_move_8through16 copy_1_move_1or2: MOVB (R11), R14 - MOVB -1(R11)(AX*1), R15 + MOVB -1(R11)(CX*1), R15 MOVB R14, (R10) - MOVB R15, -1(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 + MOVB R15, -1(R10)(CX*1) + ADDQ CX, R11 + ADDQ CX, R10 JMP copy_1_end copy_1_move_3: - MOVW (R11), R14 - MOVB 2(R11), R15 - MOVW R14, (R10) - MOVB R15, 2(R10) - ADDQ AX, R11 - ADDQ AX, R10 - JMP copy_1_end + MOVWQZX (R11), R14 + MOVB 2(R11), R15 + MOVW R14, (R10) + MOVB R15, 2(R10) + ADDQ CX, R11 + ADDQ CX, R10 + JMP copy_1_end copy_1_move_4through7: MOVL (R11), R14 - MOVL -4(R11)(AX*1), R15 + MOVL -4(R11)(CX*1), R15 MOVL R14, (R10) - MOVL R15, -4(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 + MOVL R15, -4(R10)(CX*1) + ADDQ CX, R11 + ADDQ CX, R10 JMP copy_1_end copy_1_move_8through16: MOVQ (R11), R14 - MOVQ -8(R11)(AX*1), R15 + MOVQ -8(R11)(CX*1), R15 MOVQ R14, (R10) - MOVQ R15, -8(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 + MOVQ R15, -8(R10)(CX*1) + ADDQ CX, R11 + ADDQ CX, R10 copy_1_end: - ADDQ AX, R12 + ADDQ CX, R12 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: - MOVQ R12, AX - ADDQ 40(SP), AX - CMPQ CX, AX + MOVQ R12, CX + ADDQ 72(SP), CX + CMPQ R13, CX JG error_match_off_too_big - CMPQ CX, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ CX, AX - SUBQ R12, AX + MOVQ R13, CX + SUBQ R12, CX JLS copy_match - MOVQ 48(SP), R14 - SUBQ AX, R14 - CMPQ R13, AX + MOVQ 80(SP), R14 + SUBQ CX, R14 + CMPQ AX, CX JG copy_all_from_history - MOVQ R13, AX - SUBQ $0x10, AX + MOVQ AX, CX + SUBQ $0x10, CX JB copy_4_small copy_4_loop: @@ -3260,54 +3290,54 @@ MOVUPS X0, (R10) ADDQ $0x10, R14 ADDQ $0x10, R10 - SUBQ $0x10, AX + SUBQ $0x10, CX JAE copy_4_loop - LEAQ 16(R14)(AX*1), R14 - LEAQ 16(R10)(AX*1), R10 + LEAQ 16(R14)(CX*1), R14 + LEAQ 16(R10)(CX*1), R10 MOVUPS -16(R14), X0 MOVUPS X0, -16(R10) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ AX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ AX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), AX - MOVB 2(R14), CL - MOVW AX, (R10) - MOVB CL, 2(R10) - ADDQ R13, R14 - ADDQ R13, R10 - JMP copy_4_end + MOVWQZX (R14), CX + MOVB 2(R14), R13 + MOVW CX, (R10) + MOVB R13, 2(R10) + ADDQ AX, R14 + ADDQ AX, R10 + JMP copy_4_end copy_4_move_4through7: - MOVL (R14), AX - MOVL -4(R14)(R13*1), CX - MOVL AX, (R10) - MOVL CX, -4(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVL (R14), CX + MOVL -4(R14)(AX*1), R13 + MOVL CX, (R10) + MOVL R13, -4(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), AX - MOVQ -8(R14)(R13*1), CX - MOVQ AX, (R10) - MOVQ CX, -8(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVQ (R14), CX + MOVQ -8(R14)(AX*1), R13 + MOVQ CX, (R10) + MOVQ R13, -8(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 copy_4_end: - ADDQ R13, R12 + ADDQ AX, R12 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ AX, R15 + MOVQ CX, R15 SUBQ $0x10, R15 JB copy_5_small @@ -3325,141 +3355,140 @@ JMP copy_5_end copy_5_small: - CMPQ AX, $0x03 + CMPQ CX, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ AX, $0x08 + CMPQ CX, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(AX*1), BP + MOVB -1(R14)(CX*1), BP MOVB R15, (R10) - MOVB BP, -1(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVB BP, -1(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R10) - MOVB BP, 2(R10) - ADDQ AX, R14 - ADDQ AX, R10 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R10) + MOVB BP, 2(R10) + ADDQ CX, R14 + ADDQ CX, R10 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(AX*1), BP + MOVL -4(R14)(CX*1), BP MOVL R15, (R10) - MOVL BP, -4(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVL BP, -4(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(AX*1), BP + MOVQ -8(R14)(CX*1), BP MOVQ R15, (R10) - MOVQ BP, -8(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVQ BP, -8(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 copy_5_end: - ADDQ AX, R12 - SUBQ AX, R13 + ADDQ CX, R12 + SUBQ CX, AX // Copy match from the current buffer copy_match: - MOVQ R10, AX - SUBQ CX, AX + MOVQ R10, CX + SUBQ R13, CX // ml <= mo - CMPQ R13, CX + CMPQ AX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R12 - MOVQ R13, CX - SUBQ $0x10, CX + ADDQ AX, R12 + MOVQ AX, R13 + SUBQ $0x10, R13 JB copy_2_small copy_2_loop: - MOVUPS (AX), X0 + MOVUPS (CX), X0 MOVUPS X0, (R10) - ADDQ $0x10, AX + ADDQ $0x10, CX ADDQ $0x10, R10 - SUBQ $0x10, CX + SUBQ $0x10, R13 JAE copy_2_loop - LEAQ 16(AX)(CX*1), AX - LEAQ 16(R10)(CX*1), R10 - MOVUPS -16(AX), X0 + LEAQ 16(CX)(R13*1), CX + LEAQ 16(R10)(R13*1), R10 + MOVUPS -16(CX), X0 MOVUPS X0, -16(R10) JMP copy_2_end copy_2_small: - CMPQ R13, $0x03 + CMPQ AX, $0x03 JE copy_2_move_3 JB copy_2_move_1or2 - CMPQ R13, $0x08 + CMPQ AX, $0x08 JB copy_2_move_4through7 JMP copy_2_move_8through16 copy_2_move_1or2: - MOVB (AX), CL - MOVB -1(AX)(R13*1), R14 - MOVB CL, (R10) - MOVB R14, -1(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 + MOVB (CX), R13 + MOVB -1(CX)(AX*1), R14 + MOVB R13, (R10) + MOVB R14, -1(R10)(AX*1) + ADDQ AX, CX + ADDQ AX, R10 JMP copy_2_end copy_2_move_3: - MOVW (AX), CX - MOVB 2(AX), R14 - MOVW CX, (R10) - MOVB R14, 2(R10) - ADDQ R13, AX - ADDQ R13, R10 - JMP copy_2_end + MOVWQZX (CX), R13 + MOVB 2(CX), R14 + MOVW R13, (R10) + MOVB R14, 2(R10) + ADDQ AX, CX + ADDQ AX, R10 + JMP copy_2_end copy_2_move_4through7: - MOVL (AX), CX - MOVL -4(AX)(R13*1), R14 - MOVL CX, (R10) - MOVL R14, -4(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 + MOVL (CX), R13 + MOVL -4(CX)(AX*1), R14 + MOVL R13, (R10) + MOVL R14, -4(R10)(AX*1) + ADDQ AX, CX + ADDQ AX, R10 JMP copy_2_end copy_2_move_8through16: - MOVQ (AX), CX - MOVQ -8(AX)(R13*1), R14 - MOVQ CX, (R10) - MOVQ R14, -8(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 + MOVQ (CX), R13 + MOVQ -8(CX)(AX*1), R14 + MOVQ R13, (R10) + MOVQ R14, -8(R10)(AX*1) + ADDQ AX, CX + ADDQ AX, R10 copy_2_end: JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R12 + ADDQ AX, R12 copy_slow_3: - MOVB (AX), CL - MOVB CL, (R10) - INCQ AX - INCQ R10 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (CX), R13 + MOVB R13, (R10) + INCQ CX + INCQ R10 + DECQ AX + JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_safe_amd64_main_loop loop_finished: @@ -3468,6 +3497,13 @@ MOVB BL, 40(AX) MOVQ SI, 32(AX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R12, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R12, 136(AX) @@ -3481,7 +3517,7 @@ // Return with match length error sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -3490,7 +3526,7 @@ // Return with match too long error sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -3498,7 +3534,7 @@ // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R12, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -3507,7 +3543,7 @@ // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -3520,9 +3556,9 @@ // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R12, 136(AX) MOVQ $0x00000005, ret+24(FP) @@ -3530,7 +3566,7 @@ // func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: BMI, BMI2, CMOV, SSE -TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $96-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -3542,26 +3578,34 @@ MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 - XORQ R9, R9 + MOVQ (CX), R9 MOVQ R9, 8(SP) + MOVQ 24(CX), R9 MOVQ R9, 16(SP) + MOVQ 48(CX), R9 MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) + XORQ R9, R9 + MOVQ R9, 40(SP) + MOVQ R9, 48(SP) + MOVQ R9, 56(SP) MOVQ 112(CX), R9 MOVQ 128(CX), R10 - MOVQ R10, 32(SP) + MOVQ R10, 64(SP) MOVQ 144(CX), R10 MOVQ 136(CX), R11 MOVQ 200(CX), R12 - MOVQ R12, 56(SP) + MOVQ R12, 88(SP) MOVQ 176(CX), R12 - MOVQ R12, 48(SP) + MOVQ R12, 80(SP) MOVQ 184(CX), CX - MOVQ CX, 40(SP) - MOVQ 40(SP), CX - ADDQ CX, 48(SP) + MOVQ CX, 72(SP) + MOVQ 72(SP), CX + ADDQ CX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R9, 32(SP) + ADDQ R9, 64(SP) // outBase += outPosition ADDQ R11, R9 @@ -3609,7 +3653,7 @@ MOVQ R8, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 8(SP) + MOVQ CX, 40(SP) // Update match length MOVQ $0x00000808, CX @@ -3622,7 +3666,7 @@ MOVQ DI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 16(SP) + MOVQ CX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ BX, $0x08 @@ -3664,14 +3708,13 @@ MOVQ SI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 24(SP) + MOVQ CX, 56(SP) // Fill bitreader for state updates MOVQ R12, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R12 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_safe_bmi2_skip_update LEAQ (SI)(DI*1), R13 ADDQ R8, R13 @@ -3689,8 +3732,7 @@ ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -3700,8 +3742,7 @@ ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -3710,14 +3751,13 @@ ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decodeSync_safe_bmi2_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ R12, $0x01 JBE sequenceDecs_decodeSync_safe_bmi2_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -3726,7 +3766,7 @@ JMP sequenceDecs_decodeSync_safe_bmi2_after_adjust sequenceDecs_decodeSync_safe_bmi2_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_safe_bmi2_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_safe_bmi2_adjust_offset_nonzero @@ -3761,14 +3801,11 @@ MOVQ R14, R13 sequenceDecs_decodeSync_safe_bmi2_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), CX - MOVQ 24(SP), R12 - LEAQ (CX)(R12*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), CX + MOVQ 56(SP), R12 MOVQ ctx+16(FP), R14 SUBQ R12, 104(R14) JS error_not_enough_literals @@ -3780,20 +3817,16 @@ JNZ sequenceDecs_decodeSync_safe_bmi2_error_match_len_ofs_mismatch sequenceDecs_decodeSync_safe_bmi2_match_len_ofs_ok: - MOVQ 24(SP), CX - MOVQ 8(SP), R12 - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ (CX)(R13*1), R14 + LEAQ (R12)(CX*1), R14 ADDQ R9, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ CX, CX + TESTQ R12, R12 JZ check_offset - MOVQ CX, R14 + MOVQ R12, R14 SUBQ $0x10, R14 JB copy_1_small @@ -3811,70 +3844,70 @@ JMP copy_1_end copy_1_small: - CMPQ CX, $0x03 + CMPQ R12, $0x03 JE copy_1_move_3 JB copy_1_move_1or2 - CMPQ CX, $0x08 + CMPQ R12, $0x08 JB copy_1_move_4through7 JMP copy_1_move_8through16 copy_1_move_1or2: MOVB (R10), R14 - MOVB -1(R10)(CX*1), R15 + MOVB -1(R10)(R12*1), R15 MOVB R14, (R9) - MOVB R15, -1(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 + MOVB R15, -1(R9)(R12*1) + ADDQ R12, R10 + ADDQ R12, R9 JMP copy_1_end copy_1_move_3: - MOVW (R10), R14 - MOVB 2(R10), R15 - MOVW R14, (R9) - MOVB R15, 2(R9) - ADDQ CX, R10 - ADDQ CX, R9 - JMP copy_1_end + MOVWQZX (R10), R14 + MOVB 2(R10), R15 + MOVW R14, (R9) + MOVB R15, 2(R9) + ADDQ R12, R10 + ADDQ R12, R9 + JMP copy_1_end copy_1_move_4through7: MOVL (R10), R14 - MOVL -4(R10)(CX*1), R15 + MOVL -4(R10)(R12*1), R15 MOVL R14, (R9) - MOVL R15, -4(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 + MOVL R15, -4(R9)(R12*1) + ADDQ R12, R10 + ADDQ R12, R9 JMP copy_1_end copy_1_move_8through16: MOVQ (R10), R14 - MOVQ -8(R10)(CX*1), R15 + MOVQ -8(R10)(R12*1), R15 MOVQ R14, (R9) - MOVQ R15, -8(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 + MOVQ R15, -8(R9)(R12*1) + ADDQ R12, R10 + ADDQ R12, R9 copy_1_end: - ADDQ CX, R11 + ADDQ R12, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: - MOVQ R11, CX - ADDQ 40(SP), CX - CMPQ R12, CX + MOVQ R11, R12 + ADDQ 72(SP), R12 + CMPQ R13, R12 JG error_match_off_too_big - CMPQ R12, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ R12, CX - SUBQ R11, CX + MOVQ R13, R12 + SUBQ R11, R12 JLS copy_match - MOVQ 48(SP), R14 - SUBQ CX, R14 - CMPQ R13, CX + MOVQ 80(SP), R14 + SUBQ R12, R14 + CMPQ CX, R12 JG copy_all_from_history - MOVQ R13, CX - SUBQ $0x10, CX + MOVQ CX, R12 + SUBQ $0x10, R12 JB copy_4_small copy_4_loop: @@ -3882,54 +3915,54 @@ MOVUPS X0, (R9) ADDQ $0x10, R14 ADDQ $0x10, R9 - SUBQ $0x10, CX + SUBQ $0x10, R12 JAE copy_4_loop - LEAQ 16(R14)(CX*1), R14 - LEAQ 16(R9)(CX*1), R9 + LEAQ 16(R14)(R12*1), R14 + LEAQ 16(R9)(R12*1), R9 MOVUPS -16(R14), X0 MOVUPS X0, -16(R9) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ CX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ CX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), CX - MOVB 2(R14), R12 - MOVW CX, (R9) - MOVB R12, 2(R9) - ADDQ R13, R14 - ADDQ R13, R9 - JMP copy_4_end + MOVWQZX (R14), R12 + MOVB 2(R14), R13 + MOVW R12, (R9) + MOVB R13, 2(R9) + ADDQ CX, R14 + ADDQ CX, R9 + JMP copy_4_end copy_4_move_4through7: - MOVL (R14), CX - MOVL -4(R14)(R13*1), R12 - MOVL CX, (R9) - MOVL R12, -4(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVL (R14), R12 + MOVL -4(R14)(CX*1), R13 + MOVL R12, (R9) + MOVL R13, -4(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), CX - MOVQ -8(R14)(R13*1), R12 - MOVQ CX, (R9) - MOVQ R12, -8(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVQ (R14), R12 + MOVQ -8(R14)(CX*1), R13 + MOVQ R12, (R9) + MOVQ R13, -8(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 copy_4_end: - ADDQ R13, R11 + ADDQ CX, R11 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ CX, R15 + MOVQ R12, R15 SUBQ $0x10, R15 JB copy_5_small @@ -3947,141 +3980,140 @@ JMP copy_5_end copy_5_small: - CMPQ CX, $0x03 + CMPQ R12, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ CX, $0x08 + CMPQ R12, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(CX*1), BP + MOVB -1(R14)(R12*1), BP MOVB R15, (R9) - MOVB BP, -1(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVB BP, -1(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R9) - MOVB BP, 2(R9) - ADDQ CX, R14 - ADDQ CX, R9 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R9) + MOVB BP, 2(R9) + ADDQ R12, R14 + ADDQ R12, R9 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(CX*1), BP + MOVL -4(R14)(R12*1), BP MOVL R15, (R9) - MOVL BP, -4(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVL BP, -4(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(CX*1), BP + MOVQ -8(R14)(R12*1), BP MOVQ R15, (R9) - MOVQ BP, -8(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVQ BP, -8(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 copy_5_end: - ADDQ CX, R11 - SUBQ CX, R13 + ADDQ R12, R11 + SUBQ R12, CX // Copy match from the current buffer copy_match: - MOVQ R9, CX - SUBQ R12, CX + MOVQ R9, R12 + SUBQ R13, R12 // ml <= mo - CMPQ R13, R12 + CMPQ CX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R11 - MOVQ R13, R12 - SUBQ $0x10, R12 + ADDQ CX, R11 + MOVQ CX, R13 + SUBQ $0x10, R13 JB copy_2_small copy_2_loop: - MOVUPS (CX), X0 + MOVUPS (R12), X0 MOVUPS X0, (R9) - ADDQ $0x10, CX + ADDQ $0x10, R12 ADDQ $0x10, R9 - SUBQ $0x10, R12 + SUBQ $0x10, R13 JAE copy_2_loop - LEAQ 16(CX)(R12*1), CX - LEAQ 16(R9)(R12*1), R9 - MOVUPS -16(CX), X0 + LEAQ 16(R12)(R13*1), R12 + LEAQ 16(R9)(R13*1), R9 + MOVUPS -16(R12), X0 MOVUPS X0, -16(R9) JMP copy_2_end copy_2_small: - CMPQ R13, $0x03 + CMPQ CX, $0x03 JE copy_2_move_3 JB copy_2_move_1or2 - CMPQ R13, $0x08 + CMPQ CX, $0x08 JB copy_2_move_4through7 JMP copy_2_move_8through16 copy_2_move_1or2: - MOVB (CX), R12 - MOVB -1(CX)(R13*1), R14 - MOVB R12, (R9) - MOVB R14, -1(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 + MOVB (R12), R13 + MOVB -1(R12)(CX*1), R14 + MOVB R13, (R9) + MOVB R14, -1(R9)(CX*1) + ADDQ CX, R12 + ADDQ CX, R9 JMP copy_2_end copy_2_move_3: - MOVW (CX), R12 - MOVB 2(CX), R14 - MOVW R12, (R9) - MOVB R14, 2(R9) - ADDQ R13, CX - ADDQ R13, R9 - JMP copy_2_end + MOVWQZX (R12), R13 + MOVB 2(R12), R14 + MOVW R13, (R9) + MOVB R14, 2(R9) + ADDQ CX, R12 + ADDQ CX, R9 + JMP copy_2_end copy_2_move_4through7: - MOVL (CX), R12 - MOVL -4(CX)(R13*1), R14 - MOVL R12, (R9) - MOVL R14, -4(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 + MOVL (R12), R13 + MOVL -4(R12)(CX*1), R14 + MOVL R13, (R9) + MOVL R14, -4(R9)(CX*1) + ADDQ CX, R12 + ADDQ CX, R9 JMP copy_2_end copy_2_move_8through16: - MOVQ (CX), R12 - MOVQ -8(CX)(R13*1), R14 - MOVQ R12, (R9) - MOVQ R14, -8(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 + MOVQ (R12), R13 + MOVQ -8(R12)(CX*1), R14 + MOVQ R13, (R9) + MOVQ R14, -8(R9)(CX*1) + ADDQ CX, R12 + ADDQ CX, R9 copy_2_end: JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R11 + ADDQ CX, R11 copy_slow_3: - MOVB (CX), R12 - MOVB R12, (R9) - INCQ CX - INCQ R9 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (R12), R13 + MOVB R13, (R9) + INCQ R12 + INCQ R9 + DECQ CX + JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_safe_bmi2_main_loop loop_finished: @@ -4090,6 +4122,13 @@ MOVB DL, 40(CX) MOVQ BX, 32(CX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R11, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R11, 136(AX) @@ -4103,7 +4142,7 @@ // Return with match length error sequenceDecs_decodeSync_safe_bmi2_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -4112,7 +4151,7 @@ // Return with match too long error sequenceDecs_decodeSync_safe_bmi2_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -4120,7 +4159,7 @@ // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R11, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -4129,7 +4168,7 @@ // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -4142,9 +4181,9 @@ // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R11, 136(AX) MOVQ $0x00000005, ret+24(FP) --- a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s @@ -1,22 +1,30 @@ // Code generated by command: go run gen.go -out ../seqdec.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. // EXPERIMENTAL arm64 output lowered from an amd64 avo program. -//go:build arm64 && !appengine && !noasm && gc && !noasm +//go:build arm64 && (!appengine && !noasm && gc && !noasm) // func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV -TEXT ·sequenceDecs_decode_arm64(SB), $8-32 +TEXT ·sequenceDecs_decode_arm64(SB), $40-32 MOVD br+8(FP), R1 MOVD 24(R1), R2 MOVBU 40(R1), R3 MOVD (R1), R0 MOVD 32(R1), R5 ADD R5, R0, R0 - MOVD R0, (RSP) + MOVD R0, 8(RSP) MOVD ctx+16(FP), R0 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 + MOVD (R0), R1 + MOVD R1, 16(RSP) + MOVD 24(R0), R1 + MOVD R1, 24(RSP) + MOVD 48(R0), R1 + MOVD R1, 32(RSP) + MOVD 96(R0), R1 + MOVD R1, 40(RSP) MOVD 104(R0), R9 MOVD s+0(FP), R0 MOVD 144(R0), R10 @@ -24,13 +32,12 @@ MOVD 160(R0), R12 sequenceDecs_decode_amd64_main_loop: - MOVD (RSP), R13 + MOVD 8(RSP), R13 // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 BLT sequenceDecs_decode_amd64_fill_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R13, R13 MOVD (R13), R2 SUB R0, R5, R5 @@ -57,51 +64,32 @@ sequenceDecs_decode_amd64_fill_end: // Update offset MOVD R8, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_amd64_of_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_amd64_of_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_amd64_of_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_amd64_of_update_zero: MOVD R0, 16(R9) // Update match length MOVD R7, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_amd64_ml_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_amd64_ml_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_amd64_ml_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_amd64_ml_update_zero: MOVD R0, 8(R9) // Fill bitreader to have enough for the remaining CMP $0x08, R5 BLT sequenceDecs_decode_amd64_fill_2_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R13, R13 MOVD (R13), R2 SUB R0, R5, R5 @@ -128,97 +116,64 @@ sequenceDecs_decode_amd64_fill_2_end: // Update literal length MOVD R6, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_amd64_ll_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_amd64_ll_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_amd64_ll_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_amd64_ll_update_zero: MOVD R0, (R9) // Fill bitreader for state updates - MOVD R13, (RSP) - MOVD R8, R0 - LSR $0x08, R0, R0 - MOVBU R0, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 - CMP $0x00, R16 - BEQ sequenceDecs_decode_amd64_skip_update + MOVD R13, 8(RSP) + UBFX $8, R8, $8, R0 + MOVD 40(RSP), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decode_amd64_skip_update // Update Literal Length State MOVBU R6, R13 LSRW $0x10, R6, R6 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - MOVB R13, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R6, R6 // Load ctx.llTable - MOVD ctx+16(FP), R1 - MOVD (R1), R1 - ADD R6<<3, R1, R15 - MOVD (R15), R6 + MOVD 16(RSP), R1 + MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R13 LSRW $0x10, R7, R7 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - MOVB R13, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R7, R7 // Load ctx.mlTable - MOVD ctx+16(FP), R1 - MOVD 24(R1), R1 - ADD R7<<3, R1, R15 - MOVD (R15), R7 + MOVD 24(RSP), R1 + MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R13 LSRW $0x10, R8, R8 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - MOVB R13, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R8, R8 // Load ctx.ofTable - MOVD ctx+16(FP), R1 - MOVD 48(R1), R1 - ADD R8<<3, R1, R15 - MOVD (R15), R8 + MOVD 32(RSP), R1 + MOVD (R1)(R8<<3), R8 sequenceDecs_decode_amd64_skip_update: // Adjust offset @@ -303,10 +258,9 @@ sequenceDecs_decode_amd64_match_len_ofs_ok: ADD $0x18, R9, R9 - MOVD ctx+16(FP), R0 - MOVD 96(R0), R16 + MOVD 40(RSP), R16 SUBS $1, R16, R16 - MOVD R16, 96(R0) + MOVD R16, 40(RSP) BPL sequenceDecs_decode_amd64_main_loop MOVD s+0(FP), R0 MOVD R10, 144(R0) @@ -318,53 +272,79 @@ MOVD R5, 32(R0) // Return success + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000000, R16 MOVD R16, ret+24(FP) RET // Return with match length error sequenceDecs_decode_amd64_error_match_len_ofs_mismatch: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000001, R16 MOVD R16, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_amd64_error_match_len_too_big: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) RET // Return with match offset too long error + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000003, R16 MOVD R16, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) RET // Return with overread error error_overread: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000006, R16 MOVD R16, ret+24(FP) RET // func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV -TEXT ·sequenceDecs_decode_56_arm64(SB), $8-32 +TEXT ·sequenceDecs_decode_56_arm64(SB), $40-32 MOVD br+8(FP), R1 MOVD 24(R1), R2 MOVBU 40(R1), R3 MOVD (R1), R0 MOVD 32(R1), R5 ADD R5, R0, R0 - MOVD R0, (RSP) + MOVD R0, 8(RSP) MOVD ctx+16(FP), R0 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 + MOVD (R0), R1 + MOVD R1, 16(RSP) + MOVD 24(R0), R1 + MOVD R1, 24(RSP) + MOVD 48(R0), R1 + MOVD R1, 32(RSP) + MOVD 96(R0), R1 + MOVD R1, 40(RSP) MOVD 104(R0), R9 MOVD s+0(FP), R0 MOVD 144(R0), R10 @@ -372,13 +352,12 @@ MOVD 160(R0), R12 sequenceDecs_decode_56_amd64_main_loop: - MOVD (RSP), R13 + MOVD 8(RSP), R13 // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 BLT sequenceDecs_decode_56_amd64_fill_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R13, R13 MOVD (R13), R2 SUB R0, R5, R5 @@ -405,139 +384,88 @@ sequenceDecs_decode_56_amd64_fill_end: // Update offset MOVD R8, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_56_amd64_of_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_56_amd64_of_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_56_amd64_of_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_56_amd64_of_update_zero: MOVD R0, 16(R9) // Update match length MOVD R7, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_56_amd64_ml_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_56_amd64_ml_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_56_amd64_ml_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_56_amd64_ml_update_zero: MOVD R0, 8(R9) // Update literal length MOVD R6, R0 - MOVD R3, R1 - MOVD R2, R14 - LSL R1, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decode_56_amd64_ll_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decode_56_amd64_ll_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decode_56_amd64_ll_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R14, R14 ADD R14, R0, R0 - -sequenceDecs_decode_56_amd64_ll_update_zero: MOVD R0, (R9) // Fill bitreader for state updates - MOVD R13, (RSP) - MOVD R8, R0 - LSR $0x08, R0, R0 - MOVBU R0, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 - CMP $0x00, R16 - BEQ sequenceDecs_decode_56_amd64_skip_update + MOVD R13, 8(RSP) + UBFX $8, R8, $8, R0 + MOVD 40(RSP), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decode_56_amd64_skip_update // Update Literal Length State MOVBU R6, R13 LSRW $0x10, R6, R6 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - MOVB R13, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R6, R6 // Load ctx.llTable - MOVD ctx+16(FP), R1 - MOVD (R1), R1 - ADD R6<<3, R1, R15 - MOVD (R15), R6 + MOVD 16(RSP), R1 + MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R13 LSRW $0x10, R7, R7 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - MOVB R13, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R7, R7 // Load ctx.mlTable - MOVD ctx+16(FP), R1 - MOVD 24(R1), R1 - ADD R7<<3, R1, R15 - MOVD (R15), R7 + MOVD 24(RSP), R1 + MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R13 LSRW $0x10, R8, R8 - ADD R13, R3, R1 - MOVD R2, R14 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R14, R14 - MOVD $0x00000001, R4 - MOVB R13, R1 - LSLW R1, R4, R4 - SUBW $1, R4, R4 - AND R4, R14, R14 + LSL R3, R2, R14 + LSR $0x01, R14, R14 + ADD R13, R3, R3 + EOR $0x3f, R13, R13 + LSR R13, R14, R14 ADD R14, R8, R8 // Load ctx.ofTable - MOVD ctx+16(FP), R1 - MOVD 48(R1), R1 - ADD R8<<3, R1, R15 - MOVD (R15), R8 + MOVD 32(RSP), R1 + MOVD (R1)(R8<<3), R8 sequenceDecs_decode_56_amd64_skip_update: // Adjust offset @@ -622,10 +550,9 @@ sequenceDecs_decode_56_amd64_match_len_ofs_ok: ADD $0x18, R9, R9 - MOVD ctx+16(FP), R0 - MOVD 96(R0), R16 + MOVD 40(RSP), R16 SUBS $1, R16, R16 - MOVD R16, 96(R0) + MOVD R16, 40(RSP) BPL sequenceDecs_decode_56_amd64_main_loop MOVD s+0(FP), R0 MOVD R10, 144(R0) @@ -637,42 +564,60 @@ MOVD R5, 32(R0) // Return success + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000000, R16 MOVD R16, ret+24(FP) RET // Return with match length error sequenceDecs_decode_56_amd64_error_match_len_ofs_mismatch: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000001, R16 MOVD R16, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_56_amd64_error_match_len_too_big: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) RET // Return with match offset too long error + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000003, R16 MOVD R16, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) RET // Return with overread error error_overread: + MOVD 40(RSP), R0 + MOVD ctx+16(FP), R1 + MOVD R0, 96(R1) MOVD $0x00000006, R16 MOVD R16, ret+24(FP) RET -// skipped sequenceDecs_decode_bmi2 (BMI2 not available on arm64) +// skipped sequenceDecs_decode_bmi2 (generic twin preferred on arm64) -// skipped sequenceDecs_decode_56_bmi2 (BMI2 not available on arm64) +// skipped sequenceDecs_decode_56_bmi2 (generic twin preferred on arm64) // func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool // Requires: SSE @@ -705,24 +650,27 @@ MOVD 8(R0), R12 // Copy literals - TST R10, R10 - BEQ check_offset - MOVD $0, R13 + FMOVQ (R5), F0 + FMOVQ F0, (R3) + CMP $0x10, R10 + BLS copy_1_end + MOVD $0x00000010, R13 copy_1: - ADD R13, R5, R15 - VLD1 (R15), [V0.B16] - ADD R13, R3, R15 - VST1 [V0.B16], (R15) - ADD $0x10, R13, R13 - CMP R10, R13 - BLO copy_1 - ADD R10, R5, R5 - ADD R10, R3, R3 - ADD R10, R6, R6 + ADD R13, R5, R15 + FMOVQ (R15), F0 + ADD R13, R3, R15 + FMOVQ F0, (R15) + ADD $0x10, R13, R13 + CMP R10, R13 + BLO copy_1 + +copy_1_end: + ADD R10, R5, R5 + ADD R10, R3, R3 + ADD R10, R6, R6 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: ADD R9, R6, R10 CMP R10, R11 BGT error_match_off_too_big @@ -742,21 +690,19 @@ BLO copy_4_small copy_4_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R13, R13 - ADD $0x10, R3, R3 - SUBS $0x10, R10, R10 - BHS copy_4_loop - ADD R10, R13, R13 - ADD $16, R13, R13 - ADD R10, R3, R3 - ADD $16, R3, R3 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_4_end + FMOVQ (R13), F0 + FMOVQ F0, (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R10, R10 + BHS copy_4_loop + ADD R10, R13, R13 + ADD $16, R13, R13 + ADD R10, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R3) + JMP copy_4_end copy_4_small: CMP $0x03, R12 @@ -766,13 +712,14 @@ JMP copy_4_move_8through16 copy_4_move_3: - MOVH (R13), R10 - MOVB 2(R13), R11 - MOVH R10, (R3) - MOVB R11, 2(R3) - ADD R12, R13, R13 - ADD R12, R3, R3 - JMP copy_4_end + MOVHU (R13), R10 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R11 + MOVH R10, (R3) + MOVB R11, 2(R3) + ADD R12, R13, R13 + ADD R12, R3, R3 + JMP copy_4_end copy_4_move_4through7: MOVWU (R13), R10 @@ -809,21 +756,19 @@ BLO copy_5_small copy_5_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R13, R13 - ADD $0x10, R3, R3 - SUBS $0x10, R14, R14 - BHS copy_5_loop - ADD R14, R13, R13 - ADD $16, R13, R13 - ADD R14, R3, R3 - ADD $16, R3, R3 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_5_end + FMOVQ (R13), F0 + FMOVQ F0, (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R3) + JMP copy_5_end copy_5_small: CMP $0x03, R10 @@ -834,24 +779,27 @@ JMP copy_5_move_8through16 copy_5_move_1or2: - MOVB (R13), R14 - ADD R10, R13, R15 - MOVB -1(R15), R4 - MOVB R14, (R3) - ADD R10, R3, R15 - MOVB R4, -1(R15) - ADD R10, R13, R13 - ADD R10, R3, R3 - JMP copy_5_end + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R10, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R3) + ADD R10, R3, R15 + MOVB R4, -1(R15) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end copy_5_move_3: - MOVH (R13), R14 - MOVB 2(R13), R4 - MOVH R14, (R3) - MOVB R4, 2(R3) - ADD R10, R13, R13 - ADD R10, R3, R3 - JMP copy_5_end + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R3) + MOVB R4, 2(R3) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 @@ -888,30 +836,40 @@ BHI copy_overlapping_match // Copy non-overlapping match - ADD R12, R6, R6 - MOVD R3, R11 - ADD R12, R3, R3 + ADD R12, R6, R6 + FMOVQ (R10), F0 + FMOVQ F0, (R3) + CMP $0x10, R12 + BHI copy_2_long + ADD R12, R3, R3 + JMP handle_loop + +copy_2_long: + ADD $16, R3, R11 + ADD R12, R3, R3 + ADD $0x10, R10, R10 + SUB $0x10, R12, R12 copy_2: - VLD1 (R10), [V0.B16] - VST1 [V0.B16], (R11) - ADD $0x10, R10, R10 - ADD $0x10, R11, R11 - SUBS $0x10, R12, R12 - BHI copy_2 - JMP handle_loop + FMOVQ (R10), F0 + FMOVQ F0, (R11) + ADD $0x10, R10, R10 + ADD $0x10, R11, R11 + SUBS $0x10, R12, R12 + BHI copy_2 + JMP handle_loop // Copy overlapping match copy_overlapping_match: ADD R12, R6, R6 copy_slow_3: - MOVB (R10), R11 - MOVB R11, (R3) - ADD $1, R10, R10 - ADD $1, R3, R3 - SUBS $1, R12, R12 - BNE copy_slow_3 + MOVBU (R10), R11 + MOVB R11, (R3) + ADD $1, R10, R10 + ADD $1, R3, R3 + SUBS $1, R12, R12 + BNE copy_slow_3 handle_loop: ADD $0x18, R0, R0 @@ -991,21 +949,19 @@ BLO copy_1_small copy_1_loop: - VLD1 (R5), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R5, R5 - ADD $0x10, R3, R3 - SUBS $0x10, R13, R13 - BHS copy_1_loop - ADD R13, R5, R5 - ADD $16, R5, R5 - ADD R13, R3, R3 - ADD $16, R3, R3 - ADD $-16, R5, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_1_end + FMOVQ (R5), F0 + FMOVQ F0, (R3) + ADD $0x10, R5, R5 + ADD $0x10, R3, R3 + SUBS $0x10, R13, R13 + BHS copy_1_loop + ADD R13, R5, R5 + ADD $16, R5, R5 + ADD R13, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R5), F0 + FMOVQ F0, -16(R3) + JMP copy_1_end copy_1_small: CMP $0x03, R10 @@ -1016,24 +972,27 @@ JMP copy_1_move_8through16 copy_1_move_1or2: - MOVB (R5), R13 - ADD R10, R5, R15 - MOVB -1(R15), R14 - MOVB R13, (R3) - ADD R10, R3, R15 - MOVB R14, -1(R15) - ADD R10, R5, R5 - ADD R10, R3, R3 - JMP copy_1_end + MOVBU (R5), R16 + BFI $0, R16, $8, R13 + ADD R10, R5, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R14 + MOVB R13, (R3) + ADD R10, R3, R15 + MOVB R14, -1(R15) + ADD R10, R5, R5 + ADD R10, R3, R3 + JMP copy_1_end copy_1_move_3: - MOVH (R5), R13 - MOVB 2(R5), R14 - MOVH R13, (R3) - MOVB R14, 2(R3) - ADD R10, R5, R5 - ADD R10, R3, R3 - JMP copy_1_end + MOVHU (R5), R13 + MOVBU 2(R5), R16 + BFI $0, R16, $8, R14 + MOVH R13, (R3) + MOVB R14, 2(R3) + ADD R10, R5, R5 + ADD R10, R3, R3 + JMP copy_1_end copy_1_move_4through7: MOVWU (R5), R13 @@ -1080,21 +1039,19 @@ BLO copy_4_small copy_4_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R13, R13 - ADD $0x10, R3, R3 - SUBS $0x10, R10, R10 - BHS copy_4_loop - ADD R10, R13, R13 - ADD $16, R13, R13 - ADD R10, R3, R3 - ADD $16, R3, R3 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_4_end + FMOVQ (R13), F0 + FMOVQ F0, (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R10, R10 + BHS copy_4_loop + ADD R10, R13, R13 + ADD $16, R13, R13 + ADD R10, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R3) + JMP copy_4_end copy_4_small: CMP $0x03, R12 @@ -1104,13 +1061,14 @@ JMP copy_4_move_8through16 copy_4_move_3: - MOVH (R13), R10 - MOVB 2(R13), R11 - MOVH R10, (R3) - MOVB R11, 2(R3) - ADD R12, R13, R13 - ADD R12, R3, R3 - JMP copy_4_end + MOVHU (R13), R10 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R11 + MOVH R10, (R3) + MOVB R11, 2(R3) + ADD R12, R13, R13 + ADD R12, R3, R3 + JMP copy_4_end copy_4_move_4through7: MOVWU (R13), R10 @@ -1147,21 +1105,19 @@ BLO copy_5_small copy_5_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R13, R13 - ADD $0x10, R3, R3 - SUBS $0x10, R14, R14 - BHS copy_5_loop - ADD R14, R13, R13 - ADD $16, R13, R13 - ADD R14, R3, R3 - ADD $16, R3, R3 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_5_end + FMOVQ (R13), F0 + FMOVQ F0, (R3) + ADD $0x10, R13, R13 + ADD $0x10, R3, R3 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R3) + JMP copy_5_end copy_5_small: CMP $0x03, R10 @@ -1172,24 +1128,27 @@ JMP copy_5_move_8through16 copy_5_move_1or2: - MOVB (R13), R14 - ADD R10, R13, R15 - MOVB -1(R15), R4 - MOVB R14, (R3) - ADD R10, R3, R15 - MOVB R4, -1(R15) - ADD R10, R13, R13 - ADD R10, R3, R3 - JMP copy_5_end + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R10, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R3) + ADD R10, R3, R15 + MOVB R4, -1(R15) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end copy_5_move_3: - MOVH (R13), R14 - MOVB 2(R13), R4 - MOVH R14, (R3) - MOVB R4, 2(R3) - ADD R10, R13, R13 - ADD R10, R3, R3 - JMP copy_5_end + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R3) + MOVB R4, 2(R3) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 @@ -1232,21 +1191,19 @@ BLO copy_2_small copy_2_loop: - VLD1 (R10), [V0.B16] - VST1 [V0.B16], (R3) - ADD $0x10, R10, R10 - ADD $0x10, R3, R3 - SUBS $0x10, R11, R11 - BHS copy_2_loop - ADD R11, R10, R10 - ADD $16, R10, R10 - ADD R11, R3, R3 - ADD $16, R3, R3 - ADD $-16, R10, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R3, R15 - VST1 [V0.B16], (R15) - JMP copy_2_end + FMOVQ (R10), F0 + FMOVQ F0, (R3) + ADD $0x10, R10, R10 + ADD $0x10, R3, R3 + SUBS $0x10, R11, R11 + BHS copy_2_loop + ADD R11, R10, R10 + ADD $16, R10, R10 + ADD R11, R3, R3 + ADD $16, R3, R3 + FMOVQ -16(R10), F0 + FMOVQ F0, -16(R3) + JMP copy_2_end copy_2_small: CMP $0x03, R12 @@ -1257,24 +1214,27 @@ JMP copy_2_move_8through16 copy_2_move_1or2: - MOVB (R10), R11 - ADD R12, R10, R15 - MOVB -1(R15), R13 - MOVB R11, (R3) - ADD R12, R3, R15 - MOVB R13, -1(R15) - ADD R12, R10, R10 - ADD R12, R3, R3 - JMP copy_2_end + MOVBU (R10), R16 + BFI $0, R16, $8, R11 + ADD R12, R10, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R13 + MOVB R11, (R3) + ADD R12, R3, R15 + MOVB R13, -1(R15) + ADD R12, R10, R10 + ADD R12, R3, R3 + JMP copy_2_end copy_2_move_3: - MOVH (R10), R11 - MOVB 2(R10), R13 - MOVH R11, (R3) - MOVB R13, 2(R3) - ADD R12, R10, R10 - ADD R12, R3, R3 - JMP copy_2_end + MOVHU (R10), R11 + MOVBU 2(R10), R16 + BFI $0, R16, $8, R13 + MOVH R11, (R3) + MOVB R13, 2(R3) + ADD R12, R10, R10 + ADD R12, R3, R3 + JMP copy_2_end copy_2_move_4through7: MOVWU (R10), R11 @@ -1305,12 +1265,12 @@ ADD R12, R6, R6 copy_slow_3: - MOVB (R10), R11 - MOVB R11, (R3) - ADD $1, R10, R10 - ADD $1, R3, R3 - SUBS $1, R12, R12 - BNE copy_slow_3 + MOVBU (R10), R11 + MOVB R11, (R3) + ADD $1, R10, R10 + ADD $1, R3, R3 + SUBS $1, R12, R12 + BNE copy_slow_3 handle_loop: ADD $0x18, R0, R0 @@ -1354,54 +1314,61 @@ // func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_arm64(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_arm64(SB), $96-32 MOVD br+8(FP), R1 MOVD 24(R1), R2 MOVBU 40(R1), R3 MOVD (R1), R0 MOVD 32(R1), R5 ADD R5, R0, R0 - MOVD R0, (RSP) + MOVD R0, 8(RSP) MOVD ctx+16(FP), R0 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 - MOVD $0, R1 - MOVD R1, 8(RSP) + MOVD (R0), R1 MOVD R1, 16(RSP) + MOVD 24(R0), R1 MOVD R1, 24(RSP) + MOVD 48(R0), R1 + MOVD R1, 32(RSP) + MOVD 96(R0), R1 + MOVD R1, 40(RSP) + MOVD $0, R1 + MOVD R1, 48(RSP) + MOVD R1, 56(RSP) + MOVD R1, 64(RSP) MOVD 112(R0), R9 MOVD 128(R0), R1 - MOVD R1, 32(RSP) + MOVD R1, 72(RSP) MOVD 144(R0), R10 MOVD 136(R0), R11 MOVD 200(R0), R1 - MOVD R1, 56(RSP) + MOVD R1, 96(RSP) MOVD 176(R0), R1 - MOVD R1, 48(RSP) + MOVD R1, 88(RSP) MOVD 184(R0), R0 - MOVD R0, 40(RSP) - MOVD 40(RSP), R0 - MOVD 48(RSP), R16 + MOVD R0, 80(RSP) + MOVD 80(RSP), R0 + MOVD 88(RSP), R16 ADD R0, R16, R16 - MOVD R16, 48(RSP) + MOVD R16, 88(RSP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - MOVD 32(RSP), R16 + MOVD 72(RSP), R16 ADD R9, R16, R16 - MOVD R16, 32(RSP) + MOVD R16, 72(RSP) // outBase += outPosition ADD R11, R9, R9 sequenceDecs_decodeSync_amd64_main_loop: - MOVD (RSP), R12 + MOVD 8(RSP), R12 // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 BLT sequenceDecs_decodeSync_amd64_fill_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R12, R12 MOVD (R12), R2 SUB R0, R5, R5 @@ -1428,51 +1395,32 @@ sequenceDecs_decodeSync_amd64_fill_end: // Update offset MOVD R8, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_amd64_of_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_amd64_of_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_amd64_of_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_amd64_of_update_zero: - MOVD R0, 8(RSP) + MOVD R0, 48(RSP) // Update match length MOVD R7, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_amd64_ml_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_amd64_ml_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_amd64_ml_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_amd64_ml_update_zero: - MOVD R0, 16(RSP) + MOVD R0, 56(RSP) // Fill bitreader to have enough for the remaining CMP $0x08, R5 BLT sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R12, R12 MOVD (R12), R2 SUB R0, R5, R5 @@ -1499,113 +1447,78 @@ sequenceDecs_decodeSync_amd64_fill_2_end: // Update literal length MOVD R6, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_amd64_ll_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_amd64_ll_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_amd64_ll_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_amd64_ll_update_zero: - MOVD R0, 24(RSP) + MOVD R0, 64(RSP) // Fill bitreader for state updates - MOVD R12, (RSP) - MOVD R8, R0 - LSR $0x08, R0, R0 - MOVBU R0, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 - CMP $0x00, R16 - BEQ sequenceDecs_decodeSync_amd64_skip_update + MOVD R12, 8(RSP) + UBFX $8, R8, $8, R0 + MOVD 40(RSP), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decodeSync_amd64_skip_update // Update Literal Length State MOVBU R6, R12 LSRW $0x10, R6, R6 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - MOVB R12, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R6, R6 // Load ctx.llTable - MOVD ctx+16(FP), R1 - MOVD (R1), R1 - ADD R6<<3, R1, R15 - MOVD (R15), R6 + MOVD 16(RSP), R1 + MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R12 LSRW $0x10, R7, R7 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - MOVB R12, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R7, R7 // Load ctx.mlTable - MOVD ctx+16(FP), R1 - MOVD 24(R1), R1 - ADD R7<<3, R1, R15 - MOVD (R15), R7 + MOVD 24(RSP), R1 + MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R12 LSRW $0x10, R8, R8 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - MOVB R12, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R8, R8 // Load ctx.ofTable - MOVD ctx+16(FP), R1 - MOVD 48(R1), R1 - ADD R8<<3, R1, R15 - MOVD (R15), R8 + MOVD 32(RSP), R1 + MOVD (R1)(R8<<3), R8 sequenceDecs_decodeSync_amd64_skip_update: // Adjust offset - MOVD s+0(FP), R1 - MOVD 8(RSP), R12 - CMP $0x01, R0 - BLS sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 - ADD $144, R1, R15 - VLD1 (R15), [V0.B16] - MOVD R12, 144(R1) - ADD $152, R1, R15 - VST1 [V0.B16], (R15) - JMP sequenceDecs_decodeSync_amd64_after_adjust + MOVD s+0(FP), R1 + MOVD 48(RSP), R12 + CMP $0x01, R0 + BLS sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 + FMOVQ 144(R1), F0 + MOVD R12, 144(R1) + FMOVQ F0, 152(R1) + JMP sequenceDecs_decodeSync_amd64_after_adjust sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0: - MOVD 24(RSP), R16 + MOVD 64(RSP), R16 CMP $0x00000000, R16 BNE sequenceDecs_decodeSync_amd64_adjust_offset_maybezero ADD $1, R12, R12 @@ -1643,16 +1556,11 @@ MOVD R13, R12 sequenceDecs_decodeSync_amd64_after_adjust: - MOVD R12, 8(RSP) + MOVD R12, 48(RSP) // Check values - MOVD 16(RSP), R0 - MOVD 24(RSP), R1 - ADD R1, R0, R13 - MOVD s+0(FP), R14 - MOVD 256(R14), R16 - ADD R13, R16, R16 - MOVD R16, 256(R14) + MOVD 56(RSP), R0 + MOVD 64(RSP), R1 MOVD ctx+16(FP), R13 MOVD 104(R13), R16 SUBS R1, R16, R16 @@ -1666,231 +1574,240 @@ BNE sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_amd64_match_len_ofs_ok: - MOVD 24(RSP), R0 - MOVD 8(RSP), R1 - MOVD 16(RSP), R12 - // Check if we have enough space in s.out - ADD R12, R0, R13 + ADD R0, R1, R13 + ADD $16, R13, R13 ADD R9, R13, R13 - MOVD 32(RSP), R16 + MOVD 72(RSP), R16 CMP R16, R13 BHI error_not_enough_space // Copy literals - TST R0, R0 - BEQ check_offset - MOVD $0, R13 + FMOVQ (R10), F0 + FMOVQ F0, (R9) + CMP $0x10, R1 + BLS copy_1_end + MOVD $0x00000010, R13 copy_1: - ADD R13, R10, R15 - VLD1 (R15), [V0.B16] - ADD R13, R9, R15 - VST1 [V0.B16], (R15) - ADD $0x10, R13, R13 - CMP R0, R13 - BLO copy_1 - ADD R0, R10, R10 - ADD R0, R9, R9 - ADD R0, R11, R11 + ADD R13, R10, R15 + FMOVQ (R15), F0 + ADD R13, R9, R15 + FMOVQ F0, (R15) + ADD $0x10, R13, R13 + CMP R1, R13 + BLO copy_1 + +copy_1_end: + ADD R1, R10, R10 + ADD R1, R9, R9 + ADD R1, R11, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVD R11, R0 - MOVD 40(RSP), R16 - ADD R16, R0, R0 - CMP R0, R1 + MOVD R11, R1 + MOVD 80(RSP), R16 + ADD R16, R1, R1 + CMP R1, R12 BGT error_match_off_too_big - MOVD 56(RSP), R16 - CMP R16, R1 + MOVD 96(RSP), R16 + CMP R16, R12 BGT error_match_off_too_big // Copy match from history - MOVD R1, R0 - SUBS R11, R0, R0 + MOVD R12, R1 + SUBS R11, R1, R1 BLS copy_match - MOVD 48(RSP), R13 - SUB R0, R13, R13 - CMP R0, R12 + MOVD 88(RSP), R13 + SUB R1, R13, R13 + CMP R1, R0 BGT copy_all_from_history - MOVD R12, R0 - SUBS $0x10, R0, R0 + MOVD R0, R1 + SUBS $0x10, R1, R1 BLO copy_4_small copy_4_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R13, R13 - ADD $0x10, R9, R9 - SUBS $0x10, R0, R0 - BHS copy_4_loop - ADD R0, R13, R13 - ADD $16, R13, R13 - ADD R0, R9, R9 - ADD $16, R9, R9 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_4_end + FMOVQ (R13), F0 + FMOVQ F0, (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R1, R1 + BHS copy_4_loop + ADD R1, R13, R13 + ADD $16, R13, R13 + ADD R1, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R9) + JMP copy_4_end copy_4_small: - CMP $0x03, R12 + CMP $0x03, R0 BEQ copy_4_move_3 - CMP $0x08, R12 + CMP $0x08, R0 BLO copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVH (R13), R0 - MOVB 2(R13), R1 - MOVH R0, (R9) - MOVB R1, 2(R9) - ADD R12, R13, R13 - ADD R12, R9, R9 - JMP copy_4_end + MOVHU (R13), R1 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R12 + MOVH R1, (R9) + MOVB R12, 2(R9) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_4_end copy_4_move_4through7: - MOVWU (R13), R0 - ADD R12, R13, R15 - MOVWU -4(R15), R1 - MOVW R0, (R9) - ADD R12, R9, R15 - MOVW R1, -4(R15) - ADD R12, R13, R13 - ADD R12, R9, R9 + MOVWU (R13), R1 + ADD R0, R13, R15 + MOVWU -4(R15), R12 + MOVW R1, (R9) + ADD R0, R9, R15 + MOVW R12, -4(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 JMP copy_4_end copy_4_move_8through16: - MOVD (R13), R0 - ADD R12, R13, R15 - MOVD -8(R15), R1 - MOVD R0, (R9) - ADD R12, R9, R15 - MOVD R1, -8(R15) - ADD R12, R13, R13 - ADD R12, R9, R9 + MOVD (R13), R1 + ADD R0, R13, R15 + MOVD -8(R15), R12 + MOVD R1, (R9) + ADD R0, R9, R15 + MOVD R12, -8(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 copy_4_end: - ADD R12, R11, R11 + ADD R0, R11, R11 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVD R0, R14 + MOVD R1, R14 SUBS $0x10, R14, R14 BLO copy_5_small copy_5_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R13, R13 - ADD $0x10, R9, R9 - SUBS $0x10, R14, R14 - BHS copy_5_loop - ADD R14, R13, R13 - ADD $16, R13, R13 - ADD R14, R9, R9 - ADD $16, R9, R9 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_5_end + FMOVQ (R13), F0 + FMOVQ F0, (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R9) + JMP copy_5_end copy_5_small: - CMP $0x03, R0 + CMP $0x03, R1 BEQ copy_5_move_3 BLO copy_5_move_1or2 - CMP $0x08, R0 + CMP $0x08, R1 BLO copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: - MOVB (R13), R14 - ADD R0, R13, R15 - MOVB -1(R15), R4 - MOVB R14, (R9) - ADD R0, R9, R15 - MOVB R4, -1(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 - JMP copy_5_end + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R1, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R9) + ADD R1, R9, R15 + MOVB R4, -1(R15) + ADD R1, R13, R13 + ADD R1, R9, R9 + JMP copy_5_end copy_5_move_3: - MOVH (R13), R14 - MOVB 2(R13), R4 - MOVH R14, (R9) - MOVB R4, 2(R9) - ADD R0, R13, R13 - ADD R0, R9, R9 - JMP copy_5_end + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R9) + MOVB R4, 2(R9) + ADD R1, R13, R13 + ADD R1, R9, R9 + JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVWU -4(R15), R4 MOVW R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVW R4, -4(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 JMP copy_5_end copy_5_move_8through16: MOVD (R13), R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVD -8(R15), R4 MOVD R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVD R4, -8(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 copy_5_end: - ADD R0, R11, R11 - SUB R0, R12, R12 + ADD R1, R11, R11 + SUB R1, R0, R0 // Copy match from the current buffer copy_match: - MOVD R9, R0 - SUB R1, R0, R0 + MOVD R9, R1 + SUB R12, R1, R1 // ml <= mo - CMP R1, R12 + CMP R12, R0 BHI copy_overlapping_match // Copy non-overlapping match - ADD R12, R11, R11 - MOVD R9, R1 - ADD R12, R9, R9 + ADD R0, R11, R11 + FMOVQ (R1), F0 + FMOVQ F0, (R9) + CMP $0x10, R0 + BHI copy_2_long + ADD R0, R9, R9 + JMP handle_loop + +copy_2_long: + ADD $16, R9, R12 + ADD R0, R9, R9 + ADD $0x10, R1, R1 + SUB $0x10, R0, R0 copy_2: - VLD1 (R0), [V0.B16] - VST1 [V0.B16], (R1) - ADD $0x10, R0, R0 - ADD $0x10, R1, R1 - SUBS $0x10, R12, R12 - BHI copy_2 - JMP handle_loop + FMOVQ (R1), F0 + FMOVQ F0, (R12) + ADD $0x10, R1, R1 + ADD $0x10, R12, R12 + SUBS $0x10, R0, R0 + BHI copy_2 + JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADD R12, R11, R11 + ADD R0, R11, R11 copy_slow_3: - MOVB (R0), R1 - MOVB R1, (R9) - ADD $1, R0, R0 - ADD $1, R9, R9 - SUBS $1, R12, R12 - BNE copy_slow_3 + MOVBU (R1), R12 + MOVB R12, (R9) + ADD $1, R1, R1 + ADD $1, R9, R9 + SUBS $1, R0, R0 + BNE copy_slow_3 handle_loop: - MOVD ctx+16(FP), R0 - MOVD 96(R0), R16 + MOVD 40(RSP), R16 SUBS $1, R16, R16 - MOVD R16, 96(R0) + MOVD R16, 40(RSP) BPL sequenceDecs_decodeSync_amd64_main_loop loop_finished: @@ -1899,6 +1816,16 @@ MOVB R3, 40(R0) MOVD R5, 32(R0) + // s.seqSize += outPosition - ctx.outPosition + MOVD ctx+16(FP), R0 + MOVD s+0(FP), R1 + MOVD R11, R2 + MOVD 136(R0), R16 + SUB R16, R2, R2 + MOVD 256(R1), R16 + ADD R2, R16, R16 + MOVD R16, 256(R1) + // Update the context MOVD ctx+16(FP), R0 MOVD R11, 136(R0) @@ -1913,7 +1840,7 @@ // Return with match length error sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: - MOVD 16(RSP), R0 + MOVD 56(RSP), R0 MOVD ctx+16(FP), R1 MOVD R0, 216(R1) MOVD $0x00000001, R16 @@ -1923,7 +1850,7 @@ // Return with match too long error sequenceDecs_decodeSync_amd64_error_match_len_too_big: MOVD ctx+16(FP), R0 - MOVD 16(RSP), R1 + MOVD 56(RSP), R1 MOVD R1, 216(R0) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) @@ -1932,7 +1859,7 @@ // Return with match offset too long error error_match_off_too_big: MOVD ctx+16(FP), R0 - MOVD 8(RSP), R1 + MOVD 48(RSP), R1 MOVD R1, 224(R0) MOVD R11, 136(R0) MOVD $0x00000003, R16 @@ -1942,7 +1869,7 @@ // Return with not enough literals error error_not_enough_literals: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 64(RSP), R1 MOVD R1, 208(R0) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) @@ -1957,67 +1884,74 @@ // Return with not enough output space error error_not_enough_space: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 64(RSP), R1 MOVD R1, 208(R0) - MOVD 16(RSP), R1 + MOVD 56(RSP), R1 MOVD R1, 216(R0) MOVD R11, 136(R0) MOVD $0x00000005, R16 MOVD R16, ret+24(FP) RET -// skipped sequenceDecs_decodeSync_bmi2 (BMI2 not available on arm64) +// skipped sequenceDecs_decodeSync_bmi2 (generic twin preferred on arm64) // func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_safe_arm64(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_safe_arm64(SB), $96-32 MOVD br+8(FP), R1 MOVD 24(R1), R2 MOVBU 40(R1), R3 MOVD (R1), R0 MOVD 32(R1), R5 ADD R5, R0, R0 - MOVD R0, (RSP) + MOVD R0, 8(RSP) MOVD ctx+16(FP), R0 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 - MOVD $0, R1 - MOVD R1, 8(RSP) + MOVD (R0), R1 MOVD R1, 16(RSP) + MOVD 24(R0), R1 MOVD R1, 24(RSP) + MOVD 48(R0), R1 + MOVD R1, 32(RSP) + MOVD 96(R0), R1 + MOVD R1, 40(RSP) + MOVD $0, R1 + MOVD R1, 48(RSP) + MOVD R1, 56(RSP) + MOVD R1, 64(RSP) MOVD 112(R0), R9 MOVD 128(R0), R1 - MOVD R1, 32(RSP) + MOVD R1, 72(RSP) MOVD 144(R0), R10 MOVD 136(R0), R11 MOVD 200(R0), R1 - MOVD R1, 56(RSP) + MOVD R1, 96(RSP) MOVD 176(R0), R1 - MOVD R1, 48(RSP) + MOVD R1, 88(RSP) MOVD 184(R0), R0 - MOVD R0, 40(RSP) - MOVD 40(RSP), R0 - MOVD 48(RSP), R16 + MOVD R0, 80(RSP) + MOVD 80(RSP), R0 + MOVD 88(RSP), R16 ADD R0, R16, R16 - MOVD R16, 48(RSP) + MOVD R16, 88(RSP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - MOVD 32(RSP), R16 + MOVD 72(RSP), R16 ADD R9, R16, R16 - MOVD R16, 32(RSP) + MOVD R16, 72(RSP) // outBase += outPosition ADD R11, R9, R9 sequenceDecs_decodeSync_safe_amd64_main_loop: - MOVD (RSP), R12 + MOVD 8(RSP), R12 // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 BLT sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R12, R12 MOVD (R12), R2 SUB R0, R5, R5 @@ -2044,51 +1978,32 @@ sequenceDecs_decodeSync_safe_amd64_fill_end: // Update offset MOVD R8, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_safe_amd64_of_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_safe_amd64_of_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_safe_amd64_of_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_safe_amd64_of_update_zero: - MOVD R0, 8(RSP) + MOVD R0, 48(RSP) // Update match length MOVD R7, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_safe_amd64_ml_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_safe_amd64_ml_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_safe_amd64_ml_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_safe_amd64_ml_update_zero: - MOVD R0, 16(RSP) + MOVD R0, 56(RSP) // Fill bitreader to have enough for the remaining CMP $0x08, R5 BLT sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte - MOVD R3, R0 - LSR $0x03, R0, R0 + LSR $0x03, R3, R0 SUB R0, R12, R12 MOVD (R12), R2 SUB R0, R5, R5 @@ -2115,113 +2030,78 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_end: // Update literal length MOVD R6, R0 - MOVD R3, R1 - MOVD R2, R13 - LSL R1, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 UBFX $8, R0, $8, R1 LSR $0x20, R0, R0 - TST R1, R1 - BEQ sequenceDecs_decodeSync_safe_amd64_ll_update_zero ADD R1, R3, R3 - CMP $0x40, R3 - BHI sequenceDecs_decodeSync_safe_amd64_ll_update_zero - CMP $0x40, R1 - BHS sequenceDecs_decodeSync_safe_amd64_ll_update_zero - NEG R1, R1 + EOR $0x3f, R1, R1 LSR R1, R13, R13 ADD R13, R0, R0 - -sequenceDecs_decodeSync_safe_amd64_ll_update_zero: - MOVD R0, 24(RSP) + MOVD R0, 64(RSP) // Fill bitreader for state updates - MOVD R12, (RSP) - MOVD R8, R0 - LSR $0x08, R0, R0 - MOVBU R0, R0 - MOVD ctx+16(FP), R1 - MOVD 96(R1), R16 - CMP $0x00, R16 - BEQ sequenceDecs_decodeSync_safe_amd64_skip_update + MOVD R12, 8(RSP) + UBFX $8, R8, $8, R0 + MOVD 40(RSP), R16 + CMP $0x00, R16 + BEQ sequenceDecs_decodeSync_safe_amd64_skip_update // Update Literal Length State MOVBU R6, R12 LSRW $0x10, R6, R6 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - MOVB R12, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R6, R6 // Load ctx.llTable - MOVD ctx+16(FP), R1 - MOVD (R1), R1 - ADD R6<<3, R1, R15 - MOVD (R15), R6 + MOVD 16(RSP), R1 + MOVD (R1)(R6<<3), R6 // Update Match Length State MOVBU R7, R12 LSRW $0x10, R7, R7 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - MOVB R12, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R7, R7 // Load ctx.mlTable - MOVD ctx+16(FP), R1 - MOVD 24(R1), R1 - ADD R7<<3, R1, R15 - MOVD (R15), R7 + MOVD 24(RSP), R1 + MOVD (R1)(R7<<3), R7 // Update Offset State MOVBU R8, R12 LSRW $0x10, R8, R8 - ADD R12, R3, R1 - MOVD R2, R13 - MOVD R1, R3 - NEG R1, R16 - ROR R16, R13, R13 - MOVD $0x00000001, R14 - MOVB R12, R1 - LSLW R1, R14, R14 - SUBW $1, R14, R14 - AND R14, R13, R13 + LSL R3, R2, R13 + LSR $0x01, R13, R13 + ADD R12, R3, R3 + EOR $0x3f, R12, R12 + LSR R12, R13, R13 ADD R13, R8, R8 // Load ctx.ofTable - MOVD ctx+16(FP), R1 - MOVD 48(R1), R1 - ADD R8<<3, R1, R15 - MOVD (R15), R8 + MOVD 32(RSP), R1 + MOVD (R1)(R8<<3), R8 sequenceDecs_decodeSync_safe_amd64_skip_update: // Adjust offset - MOVD s+0(FP), R1 - MOVD 8(RSP), R12 - CMP $0x01, R0 - BLS sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 - ADD $144, R1, R15 - VLD1 (R15), [V0.B16] - MOVD R12, 144(R1) - ADD $152, R1, R15 - VST1 [V0.B16], (R15) - JMP sequenceDecs_decodeSync_safe_amd64_after_adjust + MOVD s+0(FP), R1 + MOVD 48(RSP), R12 + CMP $0x01, R0 + BLS sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 + FMOVQ 144(R1), F0 + MOVD R12, 144(R1) + FMOVQ F0, 152(R1) + JMP sequenceDecs_decodeSync_safe_amd64_after_adjust sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0: - MOVD 24(RSP), R16 + MOVD 64(RSP), R16 CMP $0x00000000, R16 BNE sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero ADD $1, R12, R12 @@ -2259,16 +2139,11 @@ MOVD R13, R12 sequenceDecs_decodeSync_safe_amd64_after_adjust: - MOVD R12, 8(RSP) + MOVD R12, 48(RSP) // Check values - MOVD 16(RSP), R0 - MOVD 24(RSP), R1 - ADD R1, R0, R13 - MOVD s+0(FP), R14 - MOVD 256(R14), R16 - ADD R13, R16, R16 - MOVD R16, 256(R14) + MOVD 56(RSP), R0 + MOVD 64(RSP), R1 MOVD ctx+16(FP), R13 MOVD 104(R13), R16 SUBS R1, R16, R16 @@ -2282,351 +2157,348 @@ BNE sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok: - MOVD 24(RSP), R0 - MOVD 8(RSP), R1 - MOVD 16(RSP), R12 - // Check if we have enough space in s.out - ADD R12, R0, R13 + ADD R0, R1, R13 ADD R9, R13, R13 - MOVD 32(RSP), R16 + MOVD 72(RSP), R16 CMP R16, R13 BHI error_not_enough_space // Copy literals - TST R0, R0 + TST R1, R1 BEQ check_offset - MOVD R0, R13 + MOVD R1, R13 SUBS $0x10, R13, R13 BLO copy_1_small copy_1_loop: - VLD1 (R10), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R10, R10 - ADD $0x10, R9, R9 - SUBS $0x10, R13, R13 - BHS copy_1_loop - ADD R13, R10, R10 - ADD $16, R10, R10 - ADD R13, R9, R9 - ADD $16, R9, R9 - ADD $-16, R10, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_1_end + FMOVQ (R10), F0 + FMOVQ F0, (R9) + ADD $0x10, R10, R10 + ADD $0x10, R9, R9 + SUBS $0x10, R13, R13 + BHS copy_1_loop + ADD R13, R10, R10 + ADD $16, R10, R10 + ADD R13, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R10), F0 + FMOVQ F0, -16(R9) + JMP copy_1_end copy_1_small: - CMP $0x03, R0 + CMP $0x03, R1 BEQ copy_1_move_3 BLO copy_1_move_1or2 - CMP $0x08, R0 + CMP $0x08, R1 BLO copy_1_move_4through7 JMP copy_1_move_8through16 copy_1_move_1or2: - MOVB (R10), R13 - ADD R0, R10, R15 - MOVB -1(R15), R14 - MOVB R13, (R9) - ADD R0, R9, R15 - MOVB R14, -1(R15) - ADD R0, R10, R10 - ADD R0, R9, R9 - JMP copy_1_end + MOVBU (R10), R16 + BFI $0, R16, $8, R13 + ADD R1, R10, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R14 + MOVB R13, (R9) + ADD R1, R9, R15 + MOVB R14, -1(R15) + ADD R1, R10, R10 + ADD R1, R9, R9 + JMP copy_1_end copy_1_move_3: - MOVH (R10), R13 - MOVB 2(R10), R14 - MOVH R13, (R9) - MOVB R14, 2(R9) - ADD R0, R10, R10 - ADD R0, R9, R9 - JMP copy_1_end + MOVHU (R10), R13 + MOVBU 2(R10), R16 + BFI $0, R16, $8, R14 + MOVH R13, (R9) + MOVB R14, 2(R9) + ADD R1, R10, R10 + ADD R1, R9, R9 + JMP copy_1_end copy_1_move_4through7: MOVWU (R10), R13 - ADD R0, R10, R15 + ADD R1, R10, R15 MOVWU -4(R15), R14 MOVW R13, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVW R14, -4(R15) - ADD R0, R10, R10 - ADD R0, R9, R9 + ADD R1, R10, R10 + ADD R1, R9, R9 JMP copy_1_end copy_1_move_8through16: MOVD (R10), R13 - ADD R0, R10, R15 + ADD R1, R10, R15 MOVD -8(R15), R14 MOVD R13, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVD R14, -8(R15) - ADD R0, R10, R10 - ADD R0, R9, R9 + ADD R1, R10, R10 + ADD R1, R9, R9 copy_1_end: - ADD R0, R11, R11 + ADD R1, R11, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: - MOVD R11, R0 - MOVD 40(RSP), R16 - ADD R16, R0, R0 - CMP R0, R1 + MOVD R11, R1 + MOVD 80(RSP), R16 + ADD R16, R1, R1 + CMP R1, R12 BGT error_match_off_too_big - MOVD 56(RSP), R16 - CMP R16, R1 + MOVD 96(RSP), R16 + CMP R16, R12 BGT error_match_off_too_big // Copy match from history - MOVD R1, R0 - SUBS R11, R0, R0 + MOVD R12, R1 + SUBS R11, R1, R1 BLS copy_match - MOVD 48(RSP), R13 - SUB R0, R13, R13 - CMP R0, R12 + MOVD 88(RSP), R13 + SUB R1, R13, R13 + CMP R1, R0 BGT copy_all_from_history - MOVD R12, R0 - SUBS $0x10, R0, R0 + MOVD R0, R1 + SUBS $0x10, R1, R1 BLO copy_4_small copy_4_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R13, R13 - ADD $0x10, R9, R9 - SUBS $0x10, R0, R0 - BHS copy_4_loop - ADD R0, R13, R13 - ADD $16, R13, R13 - ADD R0, R9, R9 - ADD $16, R9, R9 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_4_end + FMOVQ (R13), F0 + FMOVQ F0, (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R1, R1 + BHS copy_4_loop + ADD R1, R13, R13 + ADD $16, R13, R13 + ADD R1, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R9) + JMP copy_4_end copy_4_small: - CMP $0x03, R12 + CMP $0x03, R0 BEQ copy_4_move_3 - CMP $0x08, R12 + CMP $0x08, R0 BLO copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVH (R13), R0 - MOVB 2(R13), R1 - MOVH R0, (R9) - MOVB R1, 2(R9) - ADD R12, R13, R13 - ADD R12, R9, R9 - JMP copy_4_end + MOVHU (R13), R1 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R12 + MOVH R1, (R9) + MOVB R12, 2(R9) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_4_end copy_4_move_4through7: - MOVWU (R13), R0 - ADD R12, R13, R15 - MOVWU -4(R15), R1 - MOVW R0, (R9) - ADD R12, R9, R15 - MOVW R1, -4(R15) - ADD R12, R13, R13 - ADD R12, R9, R9 + MOVWU (R13), R1 + ADD R0, R13, R15 + MOVWU -4(R15), R12 + MOVW R1, (R9) + ADD R0, R9, R15 + MOVW R12, -4(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 JMP copy_4_end copy_4_move_8through16: - MOVD (R13), R0 - ADD R12, R13, R15 - MOVD -8(R15), R1 - MOVD R0, (R9) - ADD R12, R9, R15 - MOVD R1, -8(R15) - ADD R12, R13, R13 - ADD R12, R9, R9 + MOVD (R13), R1 + ADD R0, R13, R15 + MOVD -8(R15), R12 + MOVD R1, (R9) + ADD R0, R9, R15 + MOVD R12, -8(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 copy_4_end: - ADD R12, R11, R11 + ADD R0, R11, R11 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVD R0, R14 + MOVD R1, R14 SUBS $0x10, R14, R14 BLO copy_5_small copy_5_loop: - VLD1 (R13), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R13, R13 - ADD $0x10, R9, R9 - SUBS $0x10, R14, R14 - BHS copy_5_loop - ADD R14, R13, R13 - ADD $16, R13, R13 - ADD R14, R9, R9 - ADD $16, R9, R9 - ADD $-16, R13, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_5_end + FMOVQ (R13), F0 + FMOVQ F0, (R9) + ADD $0x10, R13, R13 + ADD $0x10, R9, R9 + SUBS $0x10, R14, R14 + BHS copy_5_loop + ADD R14, R13, R13 + ADD $16, R13, R13 + ADD R14, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R13), F0 + FMOVQ F0, -16(R9) + JMP copy_5_end copy_5_small: - CMP $0x03, R0 + CMP $0x03, R1 BEQ copy_5_move_3 BLO copy_5_move_1or2 - CMP $0x08, R0 + CMP $0x08, R1 BLO copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: - MOVB (R13), R14 - ADD R0, R13, R15 - MOVB -1(R15), R4 - MOVB R14, (R9) - ADD R0, R9, R15 - MOVB R4, -1(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 - JMP copy_5_end + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R1, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R9) + ADD R1, R9, R15 + MOVB R4, -1(R15) + ADD R1, R13, R13 + ADD R1, R9, R9 + JMP copy_5_end copy_5_move_3: - MOVH (R13), R14 - MOVB 2(R13), R4 - MOVH R14, (R9) - MOVB R4, 2(R9) - ADD R0, R13, R13 - ADD R0, R9, R9 - JMP copy_5_end + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R9) + MOVB R4, 2(R9) + ADD R1, R13, R13 + ADD R1, R9, R9 + JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVWU -4(R15), R4 MOVW R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVW R4, -4(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 JMP copy_5_end copy_5_move_8through16: MOVD (R13), R14 - ADD R0, R13, R15 + ADD R1, R13, R15 MOVD -8(R15), R4 MOVD R14, (R9) - ADD R0, R9, R15 + ADD R1, R9, R15 MOVD R4, -8(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 + ADD R1, R13, R13 + ADD R1, R9, R9 copy_5_end: - ADD R0, R11, R11 - SUB R0, R12, R12 + ADD R1, R11, R11 + SUB R1, R0, R0 // Copy match from the current buffer copy_match: - MOVD R9, R0 - SUB R1, R0, R0 + MOVD R9, R1 + SUB R12, R1, R1 // ml <= mo - CMP R1, R12 + CMP R12, R0 BHI copy_overlapping_match // Copy non-overlapping match - ADD R12, R11, R11 - MOVD R12, R1 - SUBS $0x10, R1, R1 + ADD R0, R11, R11 + MOVD R0, R12 + SUBS $0x10, R12, R12 BLO copy_2_small copy_2_loop: - VLD1 (R0), [V0.B16] - VST1 [V0.B16], (R9) - ADD $0x10, R0, R0 - ADD $0x10, R9, R9 - SUBS $0x10, R1, R1 - BHS copy_2_loop - ADD R1, R0, R0 - ADD $16, R0, R0 - ADD R1, R9, R9 - ADD $16, R9, R9 - ADD $-16, R0, R15 - VLD1 (R15), [V0.B16] - ADD $-16, R9, R15 - VST1 [V0.B16], (R15) - JMP copy_2_end + FMOVQ (R1), F0 + FMOVQ F0, (R9) + ADD $0x10, R1, R1 + ADD $0x10, R9, R9 + SUBS $0x10, R12, R12 + BHS copy_2_loop + ADD R12, R1, R1 + ADD $16, R1, R1 + ADD R12, R9, R9 + ADD $16, R9, R9 + FMOVQ -16(R1), F0 + FMOVQ F0, -16(R9) + JMP copy_2_end copy_2_small: - CMP $0x03, R12 + CMP $0x03, R0 BEQ copy_2_move_3 BLO copy_2_move_1or2 - CMP $0x08, R12 + CMP $0x08, R0 BLO copy_2_move_4through7 JMP copy_2_move_8through16 copy_2_move_1or2: - MOVB (R0), R1 - ADD R12, R0, R15 - MOVB -1(R15), R13 - MOVB R1, (R9) - ADD R12, R9, R15 - MOVB R13, -1(R15) - ADD R12, R0, R0 - ADD R12, R9, R9 - JMP copy_2_end + MOVBU (R1), R16 + BFI $0, R16, $8, R12 + ADD R0, R1, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R13 + MOVB R12, (R9) + ADD R0, R9, R15 + MOVB R13, -1(R15) + ADD R0, R1, R1 + ADD R0, R9, R9 + JMP copy_2_end copy_2_move_3: - MOVH (R0), R1 - MOVB 2(R0), R13 - MOVH R1, (R9) - MOVB R13, 2(R9) - ADD R12, R0, R0 - ADD R12, R9, R9 - JMP copy_2_end + MOVHU (R1), R12 + MOVBU 2(R1), R16 + BFI $0, R16, $8, R13 + MOVH R12, (R9) + MOVB R13, 2(R9) + ADD R0, R1, R1 + ADD R0, R9, R9 + JMP copy_2_end copy_2_move_4through7: - MOVWU (R0), R1 - ADD R12, R0, R15 + MOVWU (R1), R12 + ADD R0, R1, R15 MOVWU -4(R15), R13 - MOVW R1, (R9) - ADD R12, R9, R15 + MOVW R12, (R9) + ADD R0, R9, R15 MOVW R13, -4(R15) - ADD R12, R0, R0 - ADD R12, R9, R9 + ADD R0, R1, R1 + ADD R0, R9, R9 JMP copy_2_end copy_2_move_8through16: - MOVD (R0), R1 - ADD R12, R0, R15 + MOVD (R1), R12 + ADD R0, R1, R15 MOVD -8(R15), R13 - MOVD R1, (R9) - ADD R12, R9, R15 + MOVD R12, (R9) + ADD R0, R9, R15 MOVD R13, -8(R15) - ADD R12, R0, R0 - ADD R12, R9, R9 + ADD R0, R1, R1 + ADD R0, R9, R9 copy_2_end: JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADD R12, R11, R11 + ADD R0, R11, R11 copy_slow_3: - MOVB (R0), R1 - MOVB R1, (R9) - ADD $1, R0, R0 - ADD $1, R9, R9 - SUBS $1, R12, R12 - BNE copy_slow_3 + MOVBU (R1), R12 + MOVB R12, (R9) + ADD $1, R1, R1 + ADD $1, R9, R9 + SUBS $1, R0, R0 + BNE copy_slow_3 handle_loop: - MOVD ctx+16(FP), R0 - MOVD 96(R0), R16 + MOVD 40(RSP), R16 SUBS $1, R16, R16 - MOVD R16, 96(R0) + MOVD R16, 40(RSP) BPL sequenceDecs_decodeSync_safe_amd64_main_loop loop_finished: @@ -2635,6 +2507,16 @@ MOVB R3, 40(R0) MOVD R5, 32(R0) + // s.seqSize += outPosition - ctx.outPosition + MOVD ctx+16(FP), R0 + MOVD s+0(FP), R1 + MOVD R11, R2 + MOVD 136(R0), R16 + SUB R16, R2, R2 + MOVD 256(R1), R16 + ADD R2, R16, R16 + MOVD R16, 256(R1) + // Update the context MOVD ctx+16(FP), R0 MOVD R11, 136(R0) @@ -2649,7 +2531,7 @@ // Return with match length error sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: - MOVD 16(RSP), R0 + MOVD 56(RSP), R0 MOVD ctx+16(FP), R1 MOVD R0, 216(R1) MOVD $0x00000001, R16 @@ -2659,7 +2541,7 @@ // Return with match too long error sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: MOVD ctx+16(FP), R0 - MOVD 16(RSP), R1 + MOVD 56(RSP), R1 MOVD R1, 216(R0) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) @@ -2668,7 +2550,7 @@ // Return with match offset too long error error_match_off_too_big: MOVD ctx+16(FP), R0 - MOVD 8(RSP), R1 + MOVD 48(RSP), R1 MOVD R1, 224(R0) MOVD R11, 136(R0) MOVD $0x00000003, R16 @@ -2678,7 +2560,7 @@ // Return with not enough literals error error_not_enough_literals: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 64(RSP), R1 MOVD R1, 208(R0) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) @@ -2693,13 +2575,13 @@ // Return with not enough output space error error_not_enough_space: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 64(RSP), R1 MOVD R1, 208(R0) - MOVD 16(RSP), R1 + MOVD 56(RSP), R1 MOVD R1, 216(R0) MOVD R11, 136(R0) MOVD $0x00000005, R16 MOVD R16, ret+24(FP) RET -// skipped sequenceDecs_decodeSync_safe_bmi2 (BMI2 not available on arm64) +// skipped sequenceDecs_decodeSync_safe_bmi2 (generic twin preferred on arm64) --- a/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go @@ -77,6 +77,36 @@ // error reported when bits are overread. const errorOverread = 6 +// useSafeDecodeSync reports whether decodeSyncSimple must use the bounds-exact +// ("safe") copy variants for the current buffer geometry. +// +// When the output and literal buffers have compressedBlockOverAlloc (16) +// bytes of slack past their logical use, the assembly may use extended +// memory copies that read and write in 16-byte blocks, overrunning the end +// of a literal run or match by up to 15 bytes; an empty literal run still +// copies one block, reading and writing the full 16 bytes of slack past +// its position. Otherwise it must use the bounds-exact ("safe") copies. +// This mirrors the analogous, always-dynamic selection in executeSimple below. +// +// The unsafe copies were disabled in #644 (2022) as a mitigation for a +// crash, but that crash's root cause — an unguarded bitReader overread that +// produced out-of-range match offsets/lengths — was fixed three days later +// in #645, which also added the fuzz corpus that has guarded this path since. +// See #1168. An asan-instrumented fuzz job (see .github/workflows/go.yml) +// covers the extended-copy path, which -race and plain fuzzing cannot. +func (s *sequenceDecs) useSafeDecodeSync() bool { + if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSizeAlloc { + return true + } + if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < int(s.maxSyncLen) { + return true + } + if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { + return true + } + return false +} + // decode sequences from the stream with the provided history but without a dictionary. func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { if len(s.dict) > 0 { @@ -86,9 +116,7 @@ return false, nil } - // FIXME: Using unsafe memory copies leads to rare, random crashes - // with fuzz testing. It is therefore disabled for now. - const useSafe = true + useSafe := s.useSafeDecodeSync() br := s.br --- a/vendor/github.com/klauspost/compress/zstd/snappy.go +++ b/vendor/github.com/klauspost/compress/zstd/snappy.go @@ -69,6 +69,10 @@ // No CRC value is being generated and not all CRC values of the Snappy stream are checked. // However, it provides really fast recompression of Snappy streams. // The converter can be reused to avoid allocations, even after errors. +// +// Deprecated: Decode the stream with s2.NewReader, which reads Snappy streams, +// and compress the output with an Encoder. This validates the input and +// compresses much better. type SnappyConverter struct { r io.Reader err error @@ -334,9 +338,10 @@ return errUnsupportedLiteralLength } - //if length > snappyMaxBlockSize-d || uint32(length) > len(src)-s { - // return ErrSnappyCorrupt - //} + if length > len(src)-s { + println("length > len(src)-s", length, len(src)-s) + return ErrSnappyCorrupt + } blk.literals = append(blk.literals, src[s:s+length]...) //println(length, "litLen") @@ -372,6 +377,14 @@ offset = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 } + // A zstd match is at least zstdMinMatch bytes. A shorter copy + // cannot be represented as a sequence and would underflow the + // matchLen computation below, producing an out-of-range match + // length code. + if length < zstdMinMatch { + return ErrSnappyCorrupt + } + if offset <= 0 || blk.size+lits < int(offset) /*|| length > len(blk)-d */ { println("offset <= 0 || blk.size+lits < int(offset)", offset, blk.size+lits, int(offset), blk.size, lits) --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -861,8 +861,8 @@ # github.com/julienschmidt/httprouter v1.3.0 ## explicit; go 1.7 github.com/julienschmidt/httprouter -# github.com/klauspost/compress v1.19.0 -## explicit; go 1.24 +# github.com/klauspost/compress v1.20.2 +## explicit; go 1.25 github.com/klauspost/compress github.com/klauspost/compress/flate github.com/klauspost/compress/fse @@ -871,6 +871,7 @@ github.com/klauspost/compress/internal/cpuinfo github.com/klauspost/compress/internal/le github.com/klauspost/compress/internal/race +github.com/klauspost/compress/internal/regmask github.com/klauspost/compress/internal/snapref github.com/klauspost/compress/s2 github.com/klauspost/compress/snappy