--- a/go.mod +++ b/go.mod @@ -64,7 +64,7 @@ github.com/jmespath/go-jmespath v0.4.0 github.com/joncrlsn/dque v0.0.0-20211108142734-c2ef48c5192a github.com/json-iterator/go v1.1.12 - github.com/klauspost/compress v1.18.7 + github.com/klauspost/compress v1.20.2 github.com/klauspost/pgzip v1.2.6 github.com/leodido/go-syslog/v4 v4.3.0 github.com/mattn/go-ieproxy v0.0.12 @@ -98,7 +98,7 @@ go.uber.org/atomic v1.11.0 go.uber.org/goleak v1.3.0 golang.org/x/crypto v0.58.0 - golang.org/x/net v0.61.0 + golang.org/x/net v0.62.0 golang.org/x/sync v0.24.0 golang.org/x/sys v0.49.0 golang.org/x/time v0.15.0 @@ -181,8 +181,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect - github.com/andybalholm/brotli v1.2.6 // indirect - github.com/apache/thrift v0.24.0 // indirect + github.com/andybalholm/brotli v1.2.7 // indirect + github.com/apache/thrift v0.25.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go v1.55.8 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 // indirect --- a/go.sum +++ b/go.sum @@ -183,14 +183,14 @@ github.com/alicebob/miniredis/v2 v2.37.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/andybalholm/brotli v1.2.6 h1:ftYnfj6usCp+UGV5kSJ3+chpMQgU+gJf/AxsUQ52REI= -github.com/andybalholm/brotli v1.2.6/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/brotli v1.2.7 h1:UgafRPGMam5ySHfRkhthMqOFA6IFoeF3D7mWIDLjG5A= +github.com/andybalholm/brotli v1.2.7/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/apache/arrow-go/v18 v18.5.2 h1:3uoHjoaEie5eVsxx/Bt64hKwZx4STb+beAkqKOlq/lY= github.com/apache/arrow-go/v18 v18.5.2/go.mod h1:yNoizNTT4peTciJ7V01d2EgOkE1d0fQ1vZcFOsVtFsw= github.com/apache/cassandra-gocql-driver/v2 v2.0.0 h1:Omnzb1Z/P90Dr2TbVNu54ICQL7TKVIIsJO231w484HU= github.com/apache/cassandra-gocql-driver/v2 v2.0.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4= -github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= -github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= +github.com/apache/thrift v0.25.0 h1:6LLlvoPjy8Cc/0b5TYRcTR3RZa2W0at3WQRcOkDwVGs= +github.com/apache/thrift v0.25.0/go.mod h1:4lwyToijD9T70w7GlevFk7FnpY60Wv8I8krMkF6FkAo= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= @@ -786,8 +786,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= -github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo= +github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -1410,8 +1410,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= -golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= --- a/vendor/github.com/andybalholm/brotli/backward_references_hq.go +++ b/vendor/github.com/andybalholm/brotli/backward_references_hq.go @@ -305,8 +305,10 @@ return uint(len) } -/* REQUIRES: nodes[pos].cost < kInfinity - REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */ +/* +REQUIRES: nodes[pos].cost < kInfinity +REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". +*/ func computeDistanceShortcut(block_start uint, pos uint, max_backward_limit uint, gap uint, nodes []zopfliNode) uint32 { var clen uint = uint(zopfliNodeCopyLength(&nodes[pos])) var ilen uint = uint(nodes[pos].dcode_insert_length & 0x7FFFFFF) @@ -326,13 +328,15 @@ } } -/* Fills in dist_cache[0..3] with the last four distances (as defined by - Section 4. of the Spec) that would be used at (block_start + pos) if we - used the shortest path of commands from block_start, computed from - nodes[0..pos]. The last four distances at block_start are in - starting_dist_cache[0..3]. - REQUIRES: nodes[pos].cost < kInfinity - REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */ +/* +Fills in dist_cache[0..3] with the last four distances (as defined by +Section 4. of the Spec) that would be used at (block_start + pos) if we +used the shortest path of commands from block_start, computed from +nodes[0..pos]. The last four distances at block_start are in +starting_dist_cache[0..3]. +REQUIRES: nodes[pos].cost < kInfinity +REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". +*/ func computeDistanceCache(pos uint, starting_dist_cache []int, nodes []zopfliNode, dist_cache []int) { var idx int = 0 var p uint = uint(nodes[pos].u.shortcut) @@ -353,8 +357,10 @@ } } -/* Maintains "ZopfliNode array invariant" and pushes node to the queue, if it - is eligible. */ +/* +Maintains "ZopfliNode array invariant" and pushes node to the queue, if it +is eligible. +*/ func evaluateNode(block_start uint, pos uint, max_backward_limit uint, gap uint, starting_dist_cache []int, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) { /* Save cost, because ComputeDistanceCache invalidates it. */ var node_cost float32 = nodes[pos].u.cost @@ -606,21 +612,24 @@ return computeShortestPathFromNodes(num_bytes, nodes) } -/* Computes the shortest path of commands from position to at most - position + num_bytes. +/* +Computes the shortest path of commands from position to at most - On return, path->size() is the number of commands found and path[i] is the - length of the i-th command (copy length plus insert length). - Note that the sum of the lengths of all commands can be less than num_bytes. - - On return, the nodes[0..num_bytes] array will have the following - "ZopfliNode array invariant": - For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then - (1) nodes[i].copy_length() >= 2 - (2) nodes[i].command_length() <= i and - (3) nodes[i - nodes[i].command_length()].cost < kInfinity + position + num_bytes. - REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 */ + On return, path->size() is the number of commands found and path[i] is the + length of the i-th command (copy length plus insert length). + Note that the sum of the lengths of all commands can be less than num_bytes. + + On return, the nodes[0..num_bytes] array will have the following + "ZopfliNode array invariant": + For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then + (1) nodes[i].copy_length() >= 2 + (2) nodes[i].command_length() <= i and + (3) nodes[i - nodes[i].command_length()].cost < kInfinity + +REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 +*/ func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint { var max_backward_limit uint = maxBackwardLimit(params.lgwin) var max_zopfli_len uint = maxZopfliLen(params) @@ -717,7 +726,6 @@ var max_length uint = num_bytes - i var num_found_matches uint var cur_match_end uint - var j uint /* Ensure that we have enough free slots. */ if matches_size < cur_match_pos+maxNumMatchesH10+shadow_matches { @@ -741,9 +749,6 @@ num_found_matches = findAllMatchesH10(hasher.(*h10), ¶ms.dictionary, ringbuffer, ringbuffer_mask, pos, max_length, max_distance, gap, params, matches[cur_match_pos+shadow_matches:]) cur_match_end = cur_match_pos + num_found_matches - for j = cur_match_pos; j+1 < cur_match_end; j++ { - assert(backwardMatchLength(&matches[j]) <= backwardMatchLength(&matches[j+1])) - } num_matches[i] = uint32(num_found_matches) if num_found_matches > 0 { --- a/vendor/github.com/andybalholm/brotli/bit_reader.go +++ b/vendor/github.com/andybalholm/brotli/bit_reader.go @@ -70,11 +70,14 @@ /* Initializes the BrotliBitReader fields. */ -/* Ensures that accumulator is not empty. - May consume up to sizeof(brotli_reg_t) - 1 bytes of input. - Returns false if data is required but there is no input available. - For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned - reading. */ +/* +Ensures that accumulator is not empty. + +May consume up to sizeof(brotli_reg_t) - 1 bytes of input. +Returns false if data is required but there is no input available. +For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned +reading. +*/ func bitReaderSaveState(from *bitReader, to *bitReaderState) { to.val_ = from.val_ to.bit_pos_ = from.bit_pos_ @@ -95,22 +98,29 @@ return 64 - br.bit_pos_ } -/* Returns amount of unread bytes the bit reader still has buffered from the - BrotliInput, including whole bytes in br->val_. */ +/* +Returns amount of unread bytes the bit reader still has buffered from the +BrotliInput, including whole bytes in br->val_. +*/ func getRemainingBytes(br *bitReader) uint { return uint(uint32(br.input_len-br.byte_pos) + (getAvailableBits(br) >> 3)) } -/* Checks if there is at least |num| bytes left in the input ring-buffer - (excluding the bits remaining in br->val_). */ +/* +Checks if there is at least |num| bytes left in the input ring-buffer +(excluding the bits remaining in br->val_). +*/ func checkInputAmount(br *bitReader, num uint) bool { return br.input_len-br.byte_pos >= num } -/* Guarantees that there are at least |n_bits| + 1 bits in accumulator. - Precondition: accumulator contains at least 1 bit. - |n_bits| should be in the range [1..24] for regular build. For portable - non-64-bit little-endian build only 16 bits are safe to request. */ +/* +Guarantees that there are at least |n_bits| + 1 bits in accumulator. + +Precondition: accumulator contains at least 1 bit. +|n_bits| should be in the range [1..24] for regular build. For portable +non-64-bit little-endian build only 16 bits are safe to request. +*/ func fillBitWindow(br *bitReader, n_bits uint32) { if br.bit_pos_ >= 32 { br.val_ >>= 32 @@ -120,14 +130,19 @@ } } -/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no - more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */ +/* +Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no +more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. +*/ func fillBitWindow16(br *bitReader) { fillBitWindow(br, 17) } -/* Tries to pull one byte of input to accumulator. - Returns false if there is no input available. */ +/* +Tries to pull one byte of input to accumulator. + +Returns false if there is no input available. +*/ func pullByte(br *bitReader) bool { if br.byte_pos == br.input_len { return false @@ -140,28 +155,38 @@ return true } -/* Returns currently available bits. - The number of valid bits could be calculated by BrotliGetAvailableBits. */ +/* +Returns currently available bits. + +The number of valid bits could be calculated by BrotliGetAvailableBits. +*/ func getBitsUnmasked(br *bitReader) uint64 { return br.val_ >> br.bit_pos_ } -/* Like BrotliGetBits, but does not mask the result. - The result contains at least 16 valid bits. */ +/* +Like BrotliGetBits, but does not mask the result. + +The result contains at least 16 valid bits. +*/ func get16BitsUnmasked(br *bitReader) uint32 { fillBitWindow(br, 16) return uint32(getBitsUnmasked(br)) } -/* Returns the specified number of bits from |br| without advancing bit - position. */ +/* +Returns the specified number of bits from |br| without advancing bit +position. +*/ func getBits(br *bitReader, n_bits uint32) uint32 { fillBitWindow(br, n_bits) return uint32(getBitsUnmasked(br)) & bitMask(n_bits) } -/* Tries to peek the specified amount of bits. Returns false, if there - is not enough input. */ +/* +Tries to peek the specified amount of bits. Returns false, if there +is not enough input. +*/ func safeGetBits(br *bitReader, n_bits uint32, val *uint32) bool { for getAvailableBits(br) < n_bits { if !pullByte(br) { @@ -191,15 +216,21 @@ br.bit_pos_ += unused_bits } -/* Reads the specified number of bits from |br| and advances the bit pos. - Precondition: accumulator MUST contain at least |n_bits|. */ +/* +Reads the specified number of bits from |br| and advances the bit pos. + +Precondition: accumulator MUST contain at least |n_bits|. +*/ func takeBits(br *bitReader, n_bits uint32, val *uint32) { *val = uint32(getBitsUnmasked(br)) & bitMask(n_bits) dropBits(br, n_bits) } -/* Reads the specified number of bits from |br| and advances the bit pos. - Assumes that there is enough input to perform BrotliFillBitWindow. */ +/* +Reads the specified number of bits from |br| and advances the bit pos. + +Assumes that there is enough input to perform BrotliFillBitWindow. +*/ func readBits(br *bitReader, n_bits uint32) uint32 { var val uint32 fillBitWindow(br, n_bits) @@ -207,8 +238,10 @@ return val } -/* Tries to read the specified amount of bits. Returns false, if there - is not enough input. |n_bits| MUST be positive. */ +/* +Tries to read the specified amount of bits. Returns false, if there +is not enough input. |n_bits| MUST be positive. +*/ func safeReadBits(br *bitReader, n_bits uint32, val *uint32) bool { for getAvailableBits(br) < n_bits { if !pullByte(br) { @@ -220,8 +253,10 @@ return true } -/* Advances the bit reader position to the next byte boundary and verifies - that any skipped bits are set to zero. */ +/* +Advances the bit reader position to the next byte boundary and verifies +that any skipped bits are set to zero. +*/ func bitReaderJumpToByteBoundary(br *bitReader) bool { var pad_bits_count uint32 = getAvailableBits(br) & 0x7 var pad_bits uint32 = 0 @@ -232,9 +267,11 @@ return pad_bits == 0 } -/* Copies remaining input bytes stored in the bit reader to the output. Value - |num| may not be larger than BrotliGetRemainingBytes. The bit reader must be - warmed up again after this. */ +/* +Copies remaining input bytes stored in the bit reader to the output. Value +|num| may not be larger than BrotliGetRemainingBytes. The bit reader must be +warmed up again after this. +*/ func copyBytes(dest []byte, br *bitReader, num uint) { for getAvailableBits(br) >= 8 && num > 0 { dest[0] = byte(getBitsUnmasked(br)) --- a/vendor/github.com/andybalholm/brotli/block_splitter_command.go +++ b/vendor/github.com/andybalholm/brotli/block_splitter_command.go @@ -51,9 +51,11 @@ } } -/* Assigns a block id from the range [0, num_histograms) to each data element - in data[0..length) and fills in block_id[0..length) with the assigned values. - Returns the number of blocks, i.e. one plus the number of block switches. */ +/* +Assigns a block id from the range [0, num_histograms) to each data element +in data[0..length) and fills in block_id[0..length) with the assigned values. +Returns the number of blocks, i.e. one plus the number of block switches. +*/ func findBlocksCommand(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramCommand, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { var data_size uint = histogramDataSizeCommand() var bitmaplen uint = (num_histograms + 7) >> 3 --- a/vendor/github.com/andybalholm/brotli/block_splitter_distance.go +++ b/vendor/github.com/andybalholm/brotli/block_splitter_distance.go @@ -51,9 +51,11 @@ } } -/* Assigns a block id from the range [0, num_histograms) to each data element - in data[0..length) and fills in block_id[0..length) with the assigned values. - Returns the number of blocks, i.e. one plus the number of block switches. */ +/* +Assigns a block id from the range [0, num_histograms) to each data element +in data[0..length) and fills in block_id[0..length) with the assigned values. +Returns the number of blocks, i.e. one plus the number of block switches. +*/ func findBlocksDistance(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramDistance, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { var data_size uint = histogramDataSizeDistance() var bitmaplen uint = (num_histograms + 7) >> 3 --- a/vendor/github.com/andybalholm/brotli/block_splitter_literal.go +++ b/vendor/github.com/andybalholm/brotli/block_splitter_literal.go @@ -51,9 +51,11 @@ } } -/* Assigns a block id from the range [0, num_histograms) to each data element - in data[0..length) and fills in block_id[0..length) with the assigned values. - Returns the number of blocks, i.e. one plus the number of block switches. */ +/* +Assigns a block id from the range [0, num_histograms) to each data element +in data[0..length) and fills in block_id[0..length) with the assigned values. +Returns the number of blocks, i.e. one plus the number of block switches. +*/ func findBlocksLiteral(data []byte, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramLiteral, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { var data_size uint = histogramDataSizeLiteral() var bitmaplen uint = (num_histograms + 7) >> 3 --- a/vendor/github.com/andybalholm/brotli/cluster_command.go +++ b/vendor/github.com/andybalholm/brotli/cluster_command.go @@ -6,8 +6,10 @@ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if - it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */ +/* +Computes the bit cost reduction by combining out[idx1] and out[idx2] and if +it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. +*/ func compareAndPushToQueueCommand(out []histogramCommand, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) { var is_good_pair bool = false var p histogramPair --- a/vendor/github.com/andybalholm/brotli/cluster_distance.go +++ b/vendor/github.com/andybalholm/brotli/cluster_distance.go @@ -8,8 +8,10 @@ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if - it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */ +/* +Computes the bit cost reduction by combining out[idx1] and out[idx2] and if +it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. +*/ func compareAndPushToQueueDistance(out []histogramDistance, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) { var is_good_pair bool = false var p histogramPair @@ -165,10 +167,12 @@ } } -/* Find the best 'out' histogram for each of the 'in' histograms. - When called, clusters[0..num_clusters) contains the unique values from - symbols[0..in_size), but this property is not preserved in this function. - Note: we assume that out[]->bit_cost_ is already up-to-date. */ +/* +Find the best 'out' histogram for each of the 'in' histograms. +When called, clusters[0..num_clusters) contains the unique values from +symbols[0..in_size), but this property is not preserved in this function. +Note: we assume that out[]->bit_cost_ is already up-to-date. +*/ func histogramRemapDistance(in []histogramDistance, in_size uint, clusters []uint32, num_clusters uint, out []histogramDistance, symbols []uint32) { var i uint for i = 0; i < in_size; i++ { --- a/vendor/github.com/andybalholm/brotli/cluster_literal.go +++ b/vendor/github.com/andybalholm/brotli/cluster_literal.go @@ -8,8 +8,10 @@ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if - it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */ +/* +Computes the bit cost reduction by combining out[idx1] and out[idx2] and if +it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. +*/ func compareAndPushToQueueLiteral(out []histogramLiteral, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) { var is_good_pair bool = false var p histogramPair @@ -165,10 +167,12 @@ } } -/* Find the best 'out' histogram for each of the 'in' histograms. - When called, clusters[0..num_clusters) contains the unique values from - symbols[0..in_size), but this property is not preserved in this function. - Note: we assume that out[]->bit_cost_ is already up-to-date. */ +/* +Find the best 'out' histogram for each of the 'in' histograms. +When called, clusters[0..num_clusters) contains the unique values from +symbols[0..in_size), but this property is not preserved in this function. +Note: we assume that out[]->bit_cost_ is already up-to-date. +*/ func histogramRemapLiteral(in []histogramLiteral, in_size uint, clusters []uint32, num_clusters uint, out []histogramLiteral, symbols []uint32) { var i uint for i = 0; i < in_size; i++ { --- a/vendor/github.com/andybalholm/brotli/compress_fragment.go +++ b/vendor/github.com/andybalholm/brotli/compress_fragment.go @@ -37,14 +37,16 @@ p1[4] == p2[4] } -/* Builds a literal prefix code into "depths" and "bits" based on the statistics - of the "input" string and stores it into the bit stream. - Note that the prefix code here is built from the pre-LZ77 input, therefore - we can only approximate the statistics of the actual literal stream. - Moreover, for long inputs we build a histogram from a sample of the input - and thus have to assign a non-zero depth for each literal. - Returns estimated compression ratio millibytes/char for encoding given input - with generated code. */ +/* +Builds a literal prefix code into "depths" and "bits" based on the statistics +of the "input" string and stores it into the bit stream. +Note that the prefix code here is built from the pre-LZ77 input, therefore +we can only approximate the statistics of the actual literal stream. +Moreover, for long inputs we build a histogram from a sample of the input +and thus have to assign a non-zero depth for each literal. +Returns estimated compression ratio millibytes/char for encoding given input +with generated code. +*/ func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte, bits []uint16, storage_ix *uint, storage []byte) uint { var histogram = [256]uint32{0} var histogram_total uint @@ -96,8 +98,10 @@ } } -/* Builds a command and distance prefix code (each 64 symbols) into "depth" and - "bits" based on "histogram" and stores it into the bit stream. */ +/* +Builds a command and distance prefix code (each 64 symbols) into "depth" and +"bits" based on "histogram" and stores it into the bit stream. +*/ func buildAndStoreCommandPrefixCode1(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { var tree [129]huffmanTree var cmd_depth = [numCommandSymbols]byte{0} @@ -785,28 +789,30 @@ } } -/* Compresses "input" string to the "*storage" buffer as one or more complete - meta-blocks, and updates the "*storage_ix" bit position. - - If "is_last" is 1, emits an additional empty last meta-block. - - "cmd_depth" and "cmd_bits" contain the command and distance prefix codes - (see comment in encode.h) used for the encoding of this input fragment. - If "is_last" is 0, they are updated to reflect the statistics - of this input fragment, to be used for the encoding of the next fragment. - - "*cmd_code_numbits" is the number of bits of the compressed representation - of the command and distance prefix codes, and "cmd_code" is an array of - at least "(*cmd_code_numbits + 7) >> 3" size that contains the compressed - command and distance prefix codes. If "is_last" is 0, these are also - updated to represent the updated "cmd_depth" and "cmd_bits". - - REQUIRES: "input_size" is greater than zero, or "is_last" is 1. - REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24). - REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. - REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two - OUTPUT: maximal copy distance <= |input_size| - OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ +/* +Compresses "input" string to the "*storage" buffer as one or more complete +meta-blocks, and updates the "*storage_ix" bit position. + +If "is_last" is 1, emits an additional empty last meta-block. + +"cmd_depth" and "cmd_bits" contain the command and distance prefix codes +(see comment in encode.h) used for the encoding of this input fragment. +If "is_last" is 0, they are updated to reflect the statistics +of this input fragment, to be used for the encoding of the next fragment. + +"*cmd_code_numbits" is the number of bits of the compressed representation +of the command and distance prefix codes, and "cmd_code" is an array of +at least "(*cmd_code_numbits + 7) >> 3" size that contains the compressed +command and distance prefix codes. If "is_last" is 0, these are also +updated to represent the updated "cmd_depth" and "cmd_bits". + +REQUIRES: "input_size" is greater than zero, or "is_last" is 1. +REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24). +REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. +REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two +OUTPUT: maximal copy distance <= |input_size| +OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) +*/ func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) { var initial_storage_ix uint = *storage_ix var table_bits uint = uint(log2FloorNonZero(table_size)) --- a/vendor/github.com/andybalholm/brotli/decode.go +++ b/vendor/github.com/andybalholm/brotli/decode.go @@ -39,6 +39,7 @@ decoderErrorFormatPadding1 = -14 decoderErrorFormatPadding2 = -15 decoderErrorFormatDistance = -16 + decoderErrorFormatBlockSwitch = -17 decoderErrorDictionaryNotSet = -19 decoderErrorInvalidArguments = -20 decoderErrorAllocContextModes = -21 @@ -54,9 +55,11 @@ const huffmanTableMask = 0xFF -/* We need the slack region for the following reasons: - - doing up to two 16-byte copies for fast backward copying - - inserting transformed dictionary word (5 prefix + 24 base + 8 suffix) */ +/* +We need the slack region for the following reasons: + - doing up to two 16-byte copies for fast backward copying + - inserting transformed dictionary word (5 prefix + 24 base + 8 suffix) +*/ const kRingBufferWriteAheadSlack uint32 = 42 var kCodeLengthCodeOrder = [codeLengthCodes]byte{1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15} @@ -84,8 +87,11 @@ } } -/* Decodes WBITS by reading 1 - 7 bits, or 0x11 for "Large Window Brotli". - Precondition: bit-reader accumulator has at least 8 bits. */ +/* +Decodes WBITS by reading 1 - 7 bits, or 0x11 for "Large Window Brotli". + +Precondition: bit-reader accumulator has at least 8 bits. +*/ func decodeWindowBits(s *Reader, br *bitReader) int { var n uint32 var large_window bool = s.large_window @@ -324,10 +330,13 @@ } } -/* Decodes the Huffman code. - This method doesn't read data from the bit reader, BUT drops the amount of - bits that correspond to the decoded symbol. - bits MUST contain at least 15 (BROTLI_HUFFMAN_MAX_CODE_LENGTH) valid bits. */ +/* +Decodes the Huffman code. + +This method doesn't read data from the bit reader, BUT drops the amount of +bits that correspond to the decoded symbol. +bits MUST contain at least 15 (BROTLI_HUFFMAN_MAX_CODE_LENGTH) valid bits. +*/ func decodeSymbol(bits uint32, table []huffmanCode, br *bitReader) uint32 { table = table[bits&huffmanTableMask:] if table[0].bits > huffmanTableBits { @@ -340,14 +349,19 @@ return uint32(table[0].value) } -/* Reads and decodes the next Huffman code from bit-stream. - This method peeks 16 bits of input and drops 0 - 15 of them. */ +/* +Reads and decodes the next Huffman code from bit-stream. + +This method peeks 16 bits of input and drops 0 - 15 of them. +*/ func readSymbol(table []huffmanCode, br *bitReader) uint32 { return decodeSymbol(get16BitsUnmasked(br), table, br) } -/* Same as DecodeSymbol, but it is known that there is less than 15 bits of - input are currently available. */ +/* +Same as DecodeSymbol, but it is known that there is less than 15 bits of +input are currently available. +*/ func safeDecodeSymbol(table []huffmanCode, br *bitReader, result *uint32) bool { var val uint32 var available_bits uint32 = getAvailableBits(br) @@ -411,8 +425,11 @@ *value = uint32(table[0].value) } -/* Decodes the next Huffman code using data prepared by PreloadSymbol. - Reads 0 - 15 bits. Also peeks 8 following bits. */ +/* +Decodes the next Huffman code using data prepared by PreloadSymbol. + +Reads 0 - 15 bits. Also peeks 8 following bits. +*/ func readPreloadedSymbol(table []huffmanCode, br *bitReader, bits *uint32, value *uint32) uint32 { var result uint32 = *value var ext []huffmanCode @@ -442,9 +459,12 @@ return result } -/* Reads (s->symbol + 1) symbols. - Totally 1..4 symbols are read, 1..11 bits each. - The list of symbols MUST NOT contain duplicates. */ +/* +Reads (s->symbol + 1) symbols. + +Totally 1..4 symbols are read, 1..11 bits each. +The list of symbols MUST NOT contain duplicates. +*/ func readSimpleHuffmanSymbols(alphabet_size uint32, max_symbol uint32, s *Reader) int { var br *bitReader = &s.br var max_bits uint32 = log2Floor(alphabet_size - 1) @@ -480,12 +500,15 @@ return decoderSuccess } -/* Process single decoded symbol code length: - A) reset the repeat variable - B) remember code length (if it is not 0) - C) extend corresponding index-chain - D) reduce the Huffman space - E) update the histogram */ +/* +Process single decoded symbol code length: + + A) reset the repeat variable + B) remember code length (if it is not 0) + C) extend corresponding index-chain + D) reduce the Huffman space + E) update the histogram +*/ func processSingleCodeLength(code_len uint32, symbol *uint32, repeat *uint32, space *uint32, prev_code_len *uint32, symbol_lists symbolList, code_length_histo []uint16, next_symbol []int) { *repeat = 0 if code_len != 0 { /* code_len == 1..15 */ @@ -499,16 +522,20 @@ (*symbol)++ } -/* Process repeated symbol code length. - A) Check if it is the extension of previous repeat sequence; if the decoded - value is not BROTLI_REPEAT_PREVIOUS_CODE_LENGTH, then it is a new - symbol-skip - B) Update repeat variable - C) Check if operation is feasible (fits alphabet) - D) For each symbol do the same operations as in ProcessSingleCodeLength +/* +Process repeated symbol code length. + + A) Check if it is the extension of previous repeat sequence; if the decoded + value is not BROTLI_REPEAT_PREVIOUS_CODE_LENGTH, then it is a new + symbol-skip + B) Update repeat variable + C) Check if operation is feasible (fits alphabet) + D) For each symbol do the same operations as in ProcessSingleCodeLength - PRECONDITION: code_len == BROTLI_REPEAT_PREVIOUS_CODE_LENGTH or - code_len == BROTLI_REPEAT_ZERO_CODE_LENGTH */ +PRECONDITION: code_len == BROTLI_REPEAT_PREVIOUS_CODE_LENGTH or + + code_len == BROTLI_REPEAT_ZERO_CODE_LENGTH +*/ func processRepeatedCodeLength(code_len uint32, repeat_delta uint32, alphabet_size uint32, symbol *uint32, repeat *uint32, space *uint32, prev_code_len *uint32, repeat_code_len *uint32, symbol_lists symbolList, code_length_histo []uint16, next_symbol []int) { var old_repeat uint32 /* for BROTLI_REPEAT_ZERO_CODE_LENGTH */ /* for BROTLI_REPEAT_ZERO_CODE_LENGTH */ var extra_bits uint32 = 3 @@ -651,8 +678,11 @@ return decoderSuccess } -/* Reads and decodes 15..18 codes using static prefix code. - Each code is 2..4 bits long. In total 30..72 bits are used. */ +/* +Reads and decodes 15..18 codes using static prefix code. + +Each code is 2..4 bits long. In total 30..72 bits are used. +*/ func readCodeLengthCodeLengths(s *Reader) int { var br *bitReader = &s.br var num_codes uint32 = s.repeat @@ -700,17 +730,21 @@ return decoderSuccess } -/* Decodes the Huffman tables. - There are 2 scenarios: - A) Huffman code contains only few symbols (1..4). Those symbols are read - directly; their code lengths are defined by the number of symbols. - For this scenario 4 - 49 bits will be read. - - B) 2-phase decoding: - B.1) Small Huffman table is decoded; it is specified with code lengths - encoded with predefined entropy code. 32 - 74 bits are used. - B.2) Decoded table is used to decode code lengths of symbols in resulting - Huffman table. In worst case 3520 bits are read. */ +/* +Decodes the Huffman tables. + +There are 2 scenarios: + + A) Huffman code contains only few symbols (1..4). Those symbols are read + directly; their code lengths are defined by the number of symbols. + For this scenario 4 - 49 bits will be read. + + B) 2-phase decoding: + B.1) Small Huffman table is decoded; it is specified with code lengths + encoded with predefined entropy code. 32 - 74 bits are used. + B.2) Decoded table is used to decode code lengths of symbols in resulting + Huffman table. In worst case 3520 bits are read. +*/ func readHuffmanCode(alphabet_size uint32, max_symbol uint32, table []huffmanCode, opt_table_size *uint32, s *Reader) int { var br *bitReader = &s.br @@ -850,8 +884,10 @@ return kBlockLengthPrefixCode[code].offset + readBits(br, nbits) } -/* WARNING: if state is not BROTLI_STATE_READ_BLOCK_LENGTH_NONE, then - reading can't be continued with ReadBlockLength. */ +/* +WARNING: if state is not BROTLI_STATE_READ_BLOCK_LENGTH_NONE, then +reading can't be continued with ReadBlockLength. +*/ func safeReadBlockLength(s *Reader, result *uint32, table []huffmanCode, br *bitReader) bool { var index uint32 if s.substate_read_block_length == stateReadBlockLengthNone { @@ -876,20 +912,24 @@ } } -/* Transform: - 1) initialize list L with values 0, 1,... 255 - 2) For each input element X: +/* +Transform: + + 1. initialize list L with values 0, 1,... 255 + + 2. For each input element X: 2.1) let Y = L[X] 2.2) remove X-th element from L 2.3) prepend Y to L 2.4) append Y to output - In most cases max(Y) <= 7, so most of L remains intact. - To reduce the cost of initialization, we reuse L, remember the upper bound - of Y values, and reinitialize only first elements in L. - - Most of input values are 0 and 1. To reduce number of branches, we replace - inner for loop with do-while. */ + In most cases max(Y) <= 7, so most of L remains intact. + To reduce the cost of initialization, we reuse L, remember the upper bound + of Y values, and reinitialize only first elements in L. + + Most of input values are 0 and 1. To reduce number of branches, we replace + inner for loop with do-while. +*/ func inverseMoveToFrontTransform(v []byte, v_len uint32, state *Reader) { var mtf [256]byte var i int @@ -936,14 +976,17 @@ return decoderSuccess } -/* Decodes a context map. - Decoding is done in 4 phases: - 1) Read auxiliary information (6..16 bits) and allocate memory. - In case of trivial context map, decoding is finished at this phase. - 2) Decode Huffman table using ReadHuffmanCode function. - This table will be used for reading context map items. - 3) Read context map items; "0" values could be run-length encoded. - 4) Optionally, apply InverseMoveToFront transform to the resulting map. */ +/* +Decodes a context map. + + Decoding is done in 4 phases: + 1) Read auxiliary information (6..16 bits) and allocate memory. + In case of trivial context map, decoding is finished at this phase. + 2) Decode Huffman table using ReadHuffmanCode function. + This table will be used for reading context map items. + 3) Read context map items; "0" values could be run-length encoded. + 4) Optionally, apply InverseMoveToFront transform to the resulting map. +*/ func decodeContextMap(context_map_size uint32, num_htrees *uint32, context_map_arg *[]byte, s *Reader) int { var br *bitReader = &s.br var result int = decoderSuccess @@ -1084,9 +1127,12 @@ } } -/* Decodes a command or literal and updates block type ring-buffer. - Reads 3..54 bits. */ -func decodeBlockTypeAndLength(safe int, s *Reader, tree_type int) bool { +/* +Decodes a command or literal and updates block type ring-buffer. + +Reads 3..54 bits. +*/ +func decodeBlockTypeAndLength(safe int, s *Reader, tree_type int) int { var max_block_type uint32 = s.num_block_types[tree_type] type_tree := s.block_type_trees[tree_type*huffmanMaxSize258:] len_tree := s.block_len_trees[tree_type*huffmanMaxSize26:] @@ -1094,7 +1140,7 @@ var ringbuffer []uint32 = s.block_type_rb[tree_type*2:] var block_type uint32 if max_block_type <= 1 { - return false + return decoderErrorFormatBlockSwitch } /* Read 0..15 + 3..39 bits. */ @@ -1105,12 +1151,12 @@ var memento bitReaderState bitReaderSaveState(br, &memento) if !safeReadSymbol(type_tree, br, &block_type) { - return false + return decoderNeedsMoreInput } if !safeReadBlockLength(s, &s.block_length[tree_type], len_tree, br) { s.substate_read_block_length = stateReadBlockLengthNone bitReaderRestoreState(br, &memento) - return false + return decoderNeedsMoreInput } } @@ -1128,7 +1174,7 @@ ringbuffer[0] = ringbuffer[1] ringbuffer[1] = block_type - return true + return decoderSuccess } func detectTrivialLiteralBlockTypes(s *Reader) { @@ -1168,61 +1214,70 @@ s.context_lookup = getContextLUT(int(context_mode)) } -/* Decodes the block type and updates the state for literal context. - Reads 3..54 bits. */ -func decodeLiteralBlockSwitchInternal(safe int, s *Reader) bool { - if !decodeBlockTypeAndLength(safe, s, 0) { - return false +/* +Decodes the block type and updates the state for literal context. + +Reads 3..54 bits. +*/ +func decodeLiteralBlockSwitchInternal(safe int, s *Reader) int { + if result := decodeBlockTypeAndLength(safe, s, 0); result != decoderSuccess { + return result } prepareLiteralDecoding(s) - return true + return decoderSuccess } -func decodeLiteralBlockSwitch(s *Reader) { - decodeLiteralBlockSwitchInternal(0, s) +func decodeLiteralBlockSwitch(s *Reader) int { + return decodeLiteralBlockSwitchInternal(0, s) } -func safeDecodeLiteralBlockSwitch(s *Reader) bool { +func safeDecodeLiteralBlockSwitch(s *Reader) int { return decodeLiteralBlockSwitchInternal(1, s) } -/* Block switch for insert/copy length. - Reads 3..54 bits. */ -func decodeCommandBlockSwitchInternal(safe int, s *Reader) bool { - if !decodeBlockTypeAndLength(safe, s, 1) { - return false +/* +Block switch for insert/copy length. + +Reads 3..54 bits. +*/ +func decodeCommandBlockSwitchInternal(safe int, s *Reader) int { + if result := decodeBlockTypeAndLength(safe, s, 1); result != decoderSuccess { + return result } s.htree_command = []huffmanCode(s.insert_copy_hgroup.htrees[s.block_type_rb[3]]) - return true + return decoderSuccess } -func decodeCommandBlockSwitch(s *Reader) { - decodeCommandBlockSwitchInternal(0, s) +func decodeCommandBlockSwitch(s *Reader) int { + return decodeCommandBlockSwitchInternal(0, s) } -func safeDecodeCommandBlockSwitch(s *Reader) bool { +func safeDecodeCommandBlockSwitch(s *Reader) int { return decodeCommandBlockSwitchInternal(1, s) } -/* Block switch for distance codes. - Reads 3..54 bits. */ -func decodeDistanceBlockSwitchInternal(safe int, s *Reader) bool { - if !decodeBlockTypeAndLength(safe, s, 2) { - return false +/* +Block switch for distance codes. + +Reads 3..54 bits. +*/ +func decodeDistanceBlockSwitchInternal(safe int, s *Reader) int { + if result := decodeBlockTypeAndLength(safe, s, 2); result != decoderSuccess { + return result } s.dist_context_map_slice = s.dist_context_map[s.block_type_rb[5]<ringbuffer_size MUST be updated by BrotliCalculateRingBufferSize before - this function is called. +s->ringbuffer_size MUST be updated by BrotliCalculateRingBufferSize before +this function is called. - Last two bytes of ring-buffer are initialized to 0, so context calculation - could be done uniformly for the first two and all other positions. */ +Last two bytes of ring-buffer are initialized to 0, so context calculation +could be done uniformly for the first two and all other positions. +*/ func ensureRingBuffer(s *Reader) bool { var old_ringbuffer []byte if s.ringbuffer_size == s.new_ringbuffer_size { @@ -1383,12 +1443,14 @@ } } -/* Calculates the smallest feasible ring buffer. +/* +Calculates the smallest feasible ring buffer. - If we know the data size is small, do not allocate more ring buffer - size than needed to reduce memory usage. + If we know the data size is small, do not allocate more ring buffer + size than needed to reduce memory usage. - When this method is called, metablock size and flags MUST be decoded. */ + When this method is called, metablock size and flags MUST be decoded. +*/ func calculateRingBufferSize(s *Reader) { var window_size int = 1 << s.window_bits var new_ringbuffer_size int = window_size @@ -1662,13 +1724,15 @@ } if s.block_length[1] == 0 { + var status int if safe != 0 { - if !safeDecodeCommandBlockSwitch(s) { - result = decoderNeedsMoreInput - goto saveStateAndReturn - } + status = safeDecodeCommandBlockSwitch(s) } else { - decodeCommandBlockSwitch(s) + status = decodeCommandBlockSwitch(s) + } + if status != decoderSuccess { + result = status + goto saveStateAndReturn } goto CommandBegin @@ -1708,13 +1772,15 @@ } if s.block_length[0] == 0 { + var status int if safe != 0 { - if !safeDecodeLiteralBlockSwitch(s) { - result = decoderNeedsMoreInput - goto saveStateAndReturn - } + status = safeDecodeLiteralBlockSwitch(s) } else { - decodeLiteralBlockSwitch(s) + status = decodeLiteralBlockSwitch(s) + } + if status != decoderSuccess { + result = status + goto saveStateAndReturn } preloadSymbol(safe, s.literal_htree, br, &bits, &value) @@ -1759,13 +1825,15 @@ } if s.block_length[0] == 0 { + var status int if safe != 0 { - if !safeDecodeLiteralBlockSwitch(s) { - result = decoderNeedsMoreInput - goto saveStateAndReturn - } + status = safeDecodeLiteralBlockSwitch(s) } else { - decodeLiteralBlockSwitch(s) + status = decodeLiteralBlockSwitch(s) + } + if status != decoderSuccess { + result = status + goto saveStateAndReturn } if s.trivial_literal_context != 0 { @@ -1826,13 +1894,15 @@ } else { /* Read distance code in the command, unless it was implicitly zero. */ if s.block_length[2] == 0 { + var status int if safe != 0 { - if !safeDecodeDistanceBlockSwitch(s) { - result = decoderNeedsMoreInput - goto saveStateAndReturn - } + status = safeDecodeDistanceBlockSwitch(s) } else { - decodeDistanceBlockSwitch(s) + status = decodeDistanceBlockSwitch(s) + } + if status != decoderSuccess { + result = status + goto saveStateAndReturn } } @@ -2011,17 +2081,19 @@ } } -/* Invariant: input stream is never overconsumed: - - invalid input implies that the whole stream is invalid -> any amount of - input could be read and discarded - - when result is "needs more input", then at least one more byte is REQUIRED - to complete decoding; all input data MUST be consumed by decoder, so - client could swap the input buffer - - when result is "needs more output" decoder MUST ensure that it doesn't - hold more than 7 bits in bit reader; this saves client from swapping input - buffer ahead of time - - when result is "success" decoder MUST return all unused data back to input - buffer; this is possible because the invariant is held on enter */ +/* +Invariant: input stream is never overconsumed: + - invalid input implies that the whole stream is invalid -> any amount of + input could be read and discarded + - when result is "needs more input", then at least one more byte is REQUIRED + to complete decoding; all input data MUST be consumed by decoder, so + client could swap the input buffer + - when result is "needs more output" decoder MUST ensure that it doesn't + hold more than 7 bits in bit reader; this saves client from swapping input + buffer ahead of time + - when result is "success" decoder MUST return all unused data back to input + buffer; this is possible because the invariant is held on enter +*/ func decoderDecompressStream(s *Reader, available_in *uint, next_in *[]byte, available_out *uint, next_out *[]byte) int { var result int = decoderSuccess var br *bitReader = &s.br @@ -2546,6 +2618,8 @@ return "PADDING_2" case decoderErrorFormatDistance: return "DISTANCE" + case decoderErrorFormatBlockSwitch: + return "BLOCK_SWITCH" case decoderErrorDictionaryNotSet: return "DICTIONARY_NOT_SET" case decoderErrorInvalidArguments: --- a/vendor/github.com/andybalholm/brotli/encode.go +++ b/vendor/github.com/andybalholm/brotli/encode.go @@ -128,8 +128,10 @@ return block_size - uint(delta) } -/* Wraps 64-bit input position to 32-bit ring-buffer position preserving - "not-a-first-lap" feature. */ +/* +Wraps 64-bit input position to 32-bit ring-buffer position preserving +"not-a-first-lap" feature. +*/ func wrapPosition(position uint64) uint32 { var result uint32 = uint32(position) var gb uint64 = position >> 30 @@ -640,11 +642,11 @@ } /* - Copies the given input data to the internal ring buffer of the compressor. - No processing of the data occurs at this time and this function can be - called multiple times before calling WriteBrotliData() to process the - accumulated input. At most input_block_size() bytes of input data can be - copied to the ring buffer, otherwise the next WriteBrotliData() will fail. +Copies the given input data to the internal ring buffer of the compressor. +No processing of the data occurs at this time and this function can be +called multiple times before calling WriteBrotliData() to process the +accumulated input. At most input_block_size() bytes of input data can be +copied to the ring buffer, otherwise the next WriteBrotliData() will fail. */ func copyInputToRingBuffer(s *Writer, input_size uint, input_buffer []byte) { var ringbuffer_ *ringBuffer = &s.ringbuffer_ @@ -699,8 +701,11 @@ } } -/* Marks all input as processed. - Returns true if position wrapping occurs. */ +/* +Marks all input as processed. + + Returns true if position wrapping occurs. +*/ func updateLastProcessedPos(s *Writer) bool { var wrapped_last_processed_pos uint32 = wrapPosition(s.last_processed_pos_) var wrapped_input_pos uint32 = wrapPosition(s.input_pos_) @@ -738,15 +743,15 @@ } /* - Processes the accumulated input data and writes - the new output meta-block to s.dest, if one has been - created (otherwise the processed input data is buffered internally). - If |is_last| or |force_flush| is true, an output meta-block is - always created. However, until |is_last| is true encoder may retain up - to 7 bits of the last byte of output. To force encoder to dump the remaining - bits use WriteMetadata() to append an empty meta-data block. - Returns false if the size of the input data is larger than - input_block_size(). +Processes the accumulated input data and writes +the new output meta-block to s.dest, if one has been +created (otherwise the processed input data is buffered internally). +If |is_last| or |force_flush| is true, an output meta-block is +always created. However, until |is_last| is true encoder may retain up +to 7 bits of the last byte of output. To force encoder to dump the remaining +bits use WriteMetadata() to append an empty meta-data block. +Returns false if the size of the input data is larger than +input_block_size(). */ func encodeData(s *Writer, is_last bool, force_flush bool) bool { var delta uint64 = unprocessedInputSize(s) @@ -915,10 +920,13 @@ } } -/* Dumps remaining output bits and metadata header to |header|. - Returns number of produced bytes. - REQUIRED: |header| should be 8-byte aligned and at least 16 bytes long. - REQUIRED: |block_size| <= (1 << 24). */ +/* +Dumps remaining output bits and metadata header to |header|. + + Returns number of produced bytes. + REQUIRED: |header| should be 8-byte aligned and at least 16 bytes long. + REQUIRED: |block_size| <= (1 << 24). +*/ func writeMetadataHeader(s *Writer, block_size uint, header []byte) uint { storage_ix := uint(s.last_bytes_bits_) header[0] = byte(s.last_bytes_) --- a/vendor/github.com/andybalholm/brotli/entropy_encode.go +++ b/vendor/github.com/andybalholm/brotli/entropy_encode.go @@ -112,21 +112,23 @@ return v0.index_right_or_value_ > v1.index_right_or_value_ } -/* This function will create a Huffman tree. +/* +This function will create a Huffman tree. - The catch here is that the tree cannot be arbitrarily deep. - Brotli specifies a maximum depth of 15 bits for "code trees" - and 7 bits for "code length code trees." + The catch here is that the tree cannot be arbitrarily deep. + Brotli specifies a maximum depth of 15 bits for "code trees" + and 7 bits for "code length code trees." + + count_limit is the value that is to be faked as the minimum value + and this minimum value is raised until the tree matches the + maximum length requirement. + + This algorithm is not of excellent performance for very long data blocks, + especially when population counts are longer than 2**tree_limit, but + we are not planning to use this with extremely long blocks. - count_limit is the value that is to be faked as the minimum value - and this minimum value is raised until the tree matches the - maximum length requirement. - - This algorithm is not of excellent performance for very long data blocks, - especially when population counts are longer than 2**tree_limit, but - we are not planning to use this with extremely long blocks. - - See http://en.wikipedia.org/wiki/Huffman_coding */ + See http://en.wikipedia.org/wiki/Huffman_coding +*/ func createHuffmanTree(data []uint32, length uint, tree_limit int, tree []huffmanTree, depth []byte) { var count_limit uint32 var sentinel huffmanTree @@ -297,13 +299,16 @@ } } -/* Change the population counts in a way that the consequent - Huffman tree compression, especially its RLE-part will be more - likely to compress this data more efficiently. - - length contains the size of the histogram. - counts contains the population counts. - good_for_rle is a buffer of at least length size */ +/* +Change the population counts in a way that the consequent + + Huffman tree compression, especially its RLE-part will be more + likely to compress this data more efficiently. + + length contains the size of the histogram. + counts contains the population counts. + good_for_rle is a buffer of at least length size +*/ func optimizeHuffmanCountsForRLE(length uint, counts []uint32, good_for_rle []byte) { var nonzero_count uint = 0 var stride uint @@ -481,9 +486,12 @@ *use_rle_for_zero = total_reps_zero > count_reps_zero*2 } -/* Write a Huffman tree from bit depths into the bit-stream representation - of a Huffman tree. The generated Huffman tree is to be compressed once - more using a Huffman tree */ +/* +Write a Huffman tree from bit depths into the bit-stream representation + + of a Huffman tree. The generated Huffman tree is to be compressed once + more using a Huffman tree +*/ func writeHuffmanTree(depth []byte, length uint, tree_size *uint, tree []byte, extra_bits_data []byte) { var previous_value byte = initialRepeatedCodeLength var i uint --- a/vendor/github.com/andybalholm/brotli/fast_log.go +++ b/vendor/github.com/andybalholm/brotli/fast_log.go @@ -17,10 +17,12 @@ return uint32(bits.Len(n)) - 1 } -/* A lookup table for small values of log2(int) to be used in entropy - computation. +/* +A lookup table for small values of log2(int) to be used in entropy +computation. - ", ".join(["%.16ff" % x for x in [0.0]+[log2(x) for x in range(1, 256)]]) */ + ", ".join(["%.16ff" % x for x in [0.0]+[log2(x) for x in range(1, 256)]]) +*/ var kLog2Table = []float32{ 0.0000000000000000, 0.0000000000000000, --- a/vendor/github.com/andybalholm/brotli/h10.go +++ b/vendor/github.com/andybalholm/brotli/h10.go @@ -24,12 +24,14 @@ return h >> (32 - 17) } -/* A (forgetful) hash table where each hash bucket contains a binary tree of - sequences whose first 4 bytes share the same hash code. - Each sequence is 128 long and is identified by its starting - position in the input data. The binary tree is sorted by the lexicographic - order of the sequences, and it is also a max-heap with respect to the - starting positions. */ +/* +A (forgetful) hash table where each hash bucket contains a binary tree of +sequences whose first 4 bytes share the same hash code. +Each sequence is 128 long and is identified by its starting +position in the input data. The binary tree is sorted by the lexicographic +order of the sequences, and it is also a max-heap with respect to the +starting positions. +*/ type h10 struct { hasherCommon window_mask_ uint @@ -61,16 +63,18 @@ return 2*(pos&self.window_mask_) + 1 } -/* Stores the hash of the next 4 bytes and in a single tree-traversal, the - hash bucket's binary tree is searched for matches and is re-rooted at the - current position. - - If less than 128 data is available, the hash bucket of the - current position is searched for matches, but the state of the hash table - is not changed, since we can not know the final sorting order of the - current (incomplete) sequence. +/* +Stores the hash of the next 4 bytes and in a single tree-traversal, the +hash bucket's binary tree is searched for matches and is re-rooted at the +current position. + +If less than 128 data is available, the hash bucket of the +current position is searched for matches, but the state of the hash table +is not changed, since we can not know the final sorting order of the +current (incomplete) sequence. - This function must be called with increasing cur_ix positions. */ +This function must be called with increasing cur_ix positions. +*/ func storeAndFindMatchesH10(self *h10, data []byte, cur_ix uint, ring_buffer_mask uint, max_length uint, max_backward uint, best_len *uint, matches []backwardMatch) []backwardMatch { var cur_ix_masked uint = cur_ix & ring_buffer_mask var max_comp_len uint = brotli_min_size_t(max_length, 128) @@ -152,13 +156,15 @@ return matches } -/* Finds all backward matches of &data[cur_ix & ring_buffer_mask] up to the - length of max_length and stores the position cur_ix in the hash table. - - Sets *num_matches to the number of matches found, and stores the found - matches in matches[0] to matches[*num_matches - 1]. The matches will be - sorted by strictly increasing length and (non-strictly) increasing - distance. */ +/* +Finds all backward matches of &data[cur_ix & ring_buffer_mask] up to the +length of max_length and stores the position cur_ix in the hash table. + +Sets *num_matches to the number of matches found, and stores the found +matches in matches[0] to matches[*num_matches - 1]. The matches will be +sorted by strictly increasing length and (non-strictly) increasing +distance. +*/ func findAllMatchesH10(handle *h10, dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, cur_ix uint, max_length uint, max_backward uint, gap uint, params *encoderParams, matches []backwardMatch) uint { var orig_matches []backwardMatch = matches var cur_ix_masked uint = cur_ix & ring_buffer_mask @@ -224,9 +230,11 @@ return uint(-cap(matches) + cap(orig_matches)) } -/* Stores the hash of the next 4 bytes and re-roots the binary tree at the - current sequence, without returning any matches. - REQUIRES: ix + 128 <= end-of-current-block */ +/* +Stores the hash of the next 4 bytes and re-roots the binary tree at the +current sequence, without returning any matches. +REQUIRES: ix + 128 <= end-of-current-block +*/ func (h *h10) Store(data []byte, mask uint, ix uint) { var max_backward uint = h.window_mask_ - windowGap + 1 /* Maximum distance is window size - 16, see section 9.1. of the spec. */ --- a/vendor/github.com/andybalholm/brotli/h5.go +++ b/vendor/github.com/andybalholm/brotli/h5.go @@ -8,12 +8,14 @@ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* A (forgetful) hash table to the data seen by the compressor, to - help create backward references to previous data. - - This is a hash map of fixed size (bucket_size_) to a ring buffer of - fixed size (block_size_). The ring buffer contains the last block_size_ - index positions of the given hash key in the compressed data. */ +/* +A (forgetful) hash table to the data seen by the compressor, to +help create backward references to previous data. + +This is a hash map of fixed size (bucket_size_) to a ring buffer of +fixed size (block_size_). The ring buffer contains the last block_size_ +index positions of the given hash key in the compressed data. +*/ func (*h5) HashTypeLength() uint { return 4 } @@ -67,8 +69,11 @@ } } -/* Look at 4 bytes at &data[ix & mask]. - Compute a hash from these, and store the value of ix at that position. */ +/* +Look at 4 bytes at &data[ix & mask]. + + Compute a hash from these, and store the value of ix at that position. +*/ func (h *h5) Store(data []byte, mask uint, ix uint) { var num []uint16 = h.num var key uint32 = hashBytesH5(data[ix&mask:], h.hash_shift_) @@ -100,17 +105,19 @@ prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check) } -/* Find a longest backward match of &data[cur_ix] up to the length of - max_length and stores the position cur_ix in the hash table. - - REQUIRES: PrepareDistanceCacheH5 must be invoked for current distance cache - values; if this method is invoked repeatedly with the same distance - cache values, it is enough to invoke PrepareDistanceCacheH5 once. - - Does not look for matches longer than max_length. - Does not look for matches further away than max_backward. - Writes the best match into |out|. - |out|->score is updated only if a better match is found. */ +/* +Find a longest backward match of &data[cur_ix] up to the length of +max_length and stores the position cur_ix in the hash table. + + REQUIRES: PrepareDistanceCacheH5 must be invoked for current distance cache + values; if this method is invoked repeatedly with the same distance + cache values, it is enough to invoke PrepareDistanceCacheH5 once. + + Does not look for matches longer than max_length. + Does not look for matches further away than max_backward. + Writes the best match into |out|. + |out|->score is updated only if a better match is found. +*/ func (h *h5) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { var num []uint16 = h.num var buckets []uint32 = h.buckets --- a/vendor/github.com/andybalholm/brotli/h6.go +++ b/vendor/github.com/andybalholm/brotli/h6.go @@ -8,12 +8,14 @@ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* A (forgetful) hash table to the data seen by the compressor, to - help create backward references to previous data. - - This is a hash map of fixed size (bucket_size_) to a ring buffer of - fixed size (block_size_). The ring buffer contains the last block_size_ - index positions of the given hash key in the compressed data. */ +/* +A (forgetful) hash table to the data seen by the compressor, to +help create backward references to previous data. + +This is a hash map of fixed size (bucket_size_) to a ring buffer of +fixed size (block_size_). The ring buffer contains the last block_size_ +index positions of the given hash key in the compressed data. +*/ func (*h6) HashTypeLength() uint { return 8 } @@ -69,8 +71,11 @@ } } -/* Look at 4 bytes at &data[ix & mask]. - Compute a hash from these, and store the value of ix at that position. */ +/* +Look at 4 bytes at &data[ix & mask]. + + Compute a hash from these, and store the value of ix at that position. +*/ func (h *h6) Store(data []byte, mask uint, ix uint) { var num []uint16 = h.num var key uint32 = hashBytesH6(data[ix&mask:], h.hash_mask_, h.hash_shift_) @@ -102,17 +107,19 @@ prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check) } -/* Find a longest backward match of &data[cur_ix] up to the length of - max_length and stores the position cur_ix in the hash table. - - REQUIRES: PrepareDistanceCacheH6 must be invoked for current distance cache - values; if this method is invoked repeatedly with the same distance - cache values, it is enough to invoke PrepareDistanceCacheH6 once. - - Does not look for matches longer than max_length. - Does not look for matches further away than max_backward. - Writes the best match into |out|. - |out|->score is updated only if a better match is found. */ +/* +Find a longest backward match of &data[cur_ix] up to the length of +max_length and stores the position cur_ix in the hash table. + + REQUIRES: PrepareDistanceCacheH6 must be invoked for current distance cache + values; if this method is invoked repeatedly with the same distance + cache values, it is enough to invoke PrepareDistanceCacheH6 once. + + Does not look for matches longer than max_length. + Does not look for matches further away than max_backward. + Writes the best match into |out|. + |out|->score is updated only if a better match is found. +*/ func (h *h6) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { var num []uint16 = h.num var buckets []uint32 = h.buckets --- a/vendor/github.com/andybalholm/brotli/hash.go +++ b/vendor/github.com/andybalholm/brotli/hash.go @@ -42,12 +42,14 @@ len_code_delta int } -/* kHashMul32 multiplier has these properties: - * The multiplier must be odd. Otherwise we may lose the highest bit. - * No long streaks of ones or zeros. - * There is no effort to ensure that it is a prime, the oddity is enough - for this use. - * The number has been tuned heuristically against compression benchmarks. */ +/* +kHashMul32 multiplier has these properties: + - The multiplier must be odd. Otherwise we may lose the highest bit. + - No long streaks of ones or zeros. + - There is no effort to ensure that it is a prime, the oddity is enough + for this use. + - The number has been tuned heuristically against compression benchmarks. +*/ const kHashMul32 uint32 = 0x1E35A7BD const kHashMul64 uint64 = 0x1E35A7BD1E35A7BD @@ -90,22 +92,24 @@ /* Score must be positive after applying maximal penalty. */ const scoreBase = (distanceBitPenalty * 8 * 8) -/* Usually, we always choose the longest backward reference. This function - allows for the exception of that rule. +/* +Usually, we always choose the longest backward reference. This function +allows for the exception of that rule. + +If we choose a backward reference that is further away, it will +usually be coded with more bits. We approximate this by assuming +log2(distance). If the distance can be expressed in terms of the +last four distances, we use some heuristic constants to estimate +the bits cost. For the first up to four literals we use the bit +cost of the literals from the literal cost model, after that we +use the average bit cost of the cost model. + +This function is used to sometimes discard a longer backward reference +when it is not much longer and the bit cost for encoding it is more +than the saved literals. - If we choose a backward reference that is further away, it will - usually be coded with more bits. We approximate this by assuming - log2(distance). If the distance can be expressed in terms of the - last four distances, we use some heuristic constants to estimate - the bits cost. For the first up to four literals we use the bit - cost of the literals from the literal cost model, after that we - use the average bit cost of the cost model. - - This function is used to sometimes discard a longer backward reference - when it is not much longer and the bit cost for encoding it is more - than the saved literals. - - backward_reference_offset MUST be positive. */ +backward_reference_offset MUST be positive. +*/ func backwardReferenceScore(copy_length uint, backward_reference_offset uint) uint { return scoreBase + literalByteScore*uint(copy_length) - distanceBitPenalty*uint(log2FloorNonZero(backward_reference_offset)) } --- a/vendor/github.com/andybalholm/brotli/hash_composite.go +++ b/vendor/github.com/andybalholm/brotli/hash_composite.go @@ -26,8 +26,10 @@ } } -/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A - and HASHER_B. */ +/* +Composite hasher: This hasher allows to combine two other hashers, HASHER_A +and HASHER_B. +*/ type hashComposite struct { hasherCommon ha hasherHandle @@ -39,10 +41,12 @@ h.params = params } -/* TODO: Initialize of the hashers is defered to Prepare (and params - remembered here) because we don't get the one_shot and input_size params - here that are needed to know the memory size of them. Instead provide - those params to all hashers InitializehashComposite */ +/* +TODO: Initialize of the hashers is defered to Prepare (and params +remembered here) because we don't get the one_shot and input_size params +here that are needed to know the memory size of them. Instead provide +those params to all hashers InitializehashComposite +*/ func (h *hashComposite) Prepare(one_shot bool, input_size uint, data []byte) { if h.ha == nil { var common_a *hasherCommon --- a/vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go +++ b/vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go @@ -22,7 +22,7 @@ /* The higher bits contain more mixture from the multiplication, so we take our results from there. */ - return uint(hash >> (32 - h.bucketBits)) + return uint(hash >> h.hash_shift) } type slot struct { @@ -30,12 +30,14 @@ next uint16 } -/* A (forgetful) hash table to the data seen by the compressor, to - help create backward references to previous data. - - Hashes are stored in chains which are bucketed to groups. Group of chains - share a storage "bank". When more than "bank size" chain nodes are added, - oldest nodes are replaced; this way several chains may share a tail. */ +/* +A (forgetful) hash table to the data seen by the compressor, to +help create backward references to previous data. + +Hashes are stored in chains which are bucketed to groups. Group of chains +share a storage "bank". When more than "bank size" chain nodes are added, +oldest nodes are replaced; this way several chains may share a tail. +*/ type hashForgetfulChain struct { hasherCommon @@ -44,6 +46,10 @@ bankBits uint numLastDistancesToCheck int + hash_shift uint + bank_mask uint + num_banks_mask uint + addr []uint32 head []uint16 tiny_hash [65536]byte @@ -60,6 +66,9 @@ q = 8 } h.max_hops = q << uint(params.quality-4) + h.hash_shift = 32 - h.bucketBits + h.bank_mask = (1 << h.bankBits) - 1 + h.num_banks_mask = h.numBanks - 1 bankSize := 1 << h.bankBits bucketSize := 1 << h.bucketBits @@ -105,12 +114,14 @@ } } -/* Look at 4 bytes at &data[ix & mask]. Compute a hash from these, and prepend - node to corresponding chain; also update tiny_hash for current position. */ +/* +Look at 4 bytes at &data[ix & mask]. Compute a hash from these, and prepend +node to corresponding chain; also update tiny_hash for current position. +*/ func (h *hashForgetfulChain) Store(data []byte, mask uint, ix uint) { var key uint = h.HashBytes(data[ix&mask:]) - var bank uint = key & (h.numBanks - 1) - idx := uint(h.free_slot_idx[bank]) & ((1 << h.bankBits) - 1) + var bank uint = key & h.num_banks_mask + idx := uint(h.free_slot_idx[bank]) & h.bank_mask h.free_slot_idx[bank]++ var delta uint = ix - uint(h.addr[key]) h.tiny_hash[uint16(ix)] = byte(key) @@ -145,17 +156,20 @@ prepareDistanceCache(distance_cache, h.numLastDistancesToCheck) } -/* Find a longest backward match of &data[cur_ix] up to the length of - max_length and stores the position cur_ix in the hash table. - - REQUIRES: PrepareDistanceCachehashForgetfulChain must be invoked for current distance cache - values; if this method is invoked repeatedly with the same distance - cache values, it is enough to invoke PrepareDistanceCachehashForgetfulChain once. - - Does not look for matches longer than max_length. - Does not look for matches further away than max_backward. - Writes the best match into |out|. - |out|->score is updated only if a better match is found. */ +/* +Find a longest backward match of &data[cur_ix] up to the length of +max_length and stores the position cur_ix in the hash table. + +REQUIRES: PrepareDistanceCachehashForgetfulChain must be invoked for current distance cache + + values; if this method is invoked repeatedly with the same distance + cache values, it is enough to invoke PrepareDistanceCachehashForgetfulChain once. + +Does not look for matches longer than max_length. +Does not look for matches further away than max_backward. +Writes the best match into |out|. +|out|->score is updated only if a better match is found. +*/ func (h *hashForgetfulChain) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { var cur_ix_masked uint = cur_ix & ring_buffer_mask var min_score uint = out.score @@ -202,7 +216,7 @@ } } { - var bank uint = key & (h.numBanks - 1) + var bank uint = key & h.num_banks_mask var backward uint = 0 var hops uint = h.max_hops var delta uint = cur_ix - uint(h.addr[key]) --- a/vendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go +++ b/vendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go @@ -20,22 +20,26 @@ return 8 } -/* HashBytes is the function that chooses the bucket to place - the address in. The HashLongestMatch and hashLongestMatchQuickly - classes have separate, different implementations of hashing. */ +/* +HashBytes is the function that chooses the bucket to place +the address in. The HashLongestMatch and hashLongestMatchQuickly +classes have separate, different implementations of hashing. +*/ func (h *hashLongestMatchQuickly) HashBytes(data []byte) uint32 { - var hash uint64 = ((binary.LittleEndian.Uint64(data) << (64 - 8*h.hashLen)) * kHashMul64) + var hash uint64 = (binary.LittleEndian.Uint64(data) << h.hash_shift_left) * kHashMul64 /* The higher bits contain more mixture from the multiplication, so we take our results from there. */ - return uint32(hash >> (64 - h.bucketBits)) + return uint32(hash >> h.hash_shift_right) } -/* A (forgetful) hash table to the data seen by the compressor, to - help create backward references to previous data. +/* +A (forgetful) hash table to the data seen by the compressor, to +help create backward references to previous data. - This is a hash map of fixed size (1 << 16). Starting from the - given index, 1 buckets are used to store values of a key. */ +This is a hash map of fixed size (1 << 16). Starting from the +given index, 1 buckets are used to store values of a key. +*/ type hashLongestMatchQuickly struct { hasherCommon @@ -44,10 +48,17 @@ hashLen uint useDictionary bool + hash_shift_left uint + hash_shift_right uint + sweep_mask uint32 + buckets []uint32 } func (h *hashLongestMatchQuickly) Initialize(params *encoderParams) { + h.hash_shift_left = 64 - 8*h.hashLen + h.hash_shift_right = 64 - h.bucketBits + h.sweep_mask = uint32(h.bucketSweep - 1) h.buckets = make([]uint32, 1<>3) % uint32(h.bucketSweep) + var off uint32 = uint32(ix>>3) & h.sweep_mask /* Wiggle the value with the bucket sweep range. */ h.buckets[key+off] = uint32(ix) } @@ -104,14 +118,16 @@ func (*hashLongestMatchQuickly) PrepareDistanceCache(distance_cache []int) { } -/* Find a longest backward match of &data[cur_ix & ring_buffer_mask] - up to the length of max_length and stores the position cur_ix in the - hash table. - - Does not look for matches longer than max_length. - Does not look for matches further away than max_backward. - Writes the best match into |out|. - |out|->score is updated only if a better match is found. */ +/* +Find a longest backward match of &data[cur_ix & ring_buffer_mask] +up to the length of max_length and stores the position cur_ix in the +hash table. + +Does not look for matches longer than max_length. +Does not look for matches further away than max_backward. +Writes the best match into |out|. +|out|->score is updated only if a better match is found. +*/ func (h *hashLongestMatchQuickly) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { var best_len_in uint = out.len var cur_ix_masked uint = cur_ix & ring_buffer_mask --- a/vendor/github.com/andybalholm/brotli/hash_rolling.go +++ b/vendor/github.com/andybalholm/brotli/hash_rolling.go @@ -13,9 +13,11 @@ const kInvalidPosHashRolling uint32 = 0xffffffff -/* This hasher uses a longer forward length, but returning a higher value here - will hurt compression by the main hasher when combined with a composite - hasher. The hasher tests for forward itself instead. */ +/* +This hasher uses a longer forward length, but returning a higher value here +will hurt compression by the main hasher when combined with a composite +hasher. The hasher tests for forward itself instead. +*/ func (*hashRolling) HashTypeLength() uint { return 4 } @@ -24,8 +26,10 @@ return 4 } -/* Computes a code from a single byte. A lookup table of 256 values could be - used, but simply adding 1 works about as good. */ +/* +Computes a code from a single byte. A lookup table of 256 values could be +used, but simply adding 1 works about as good. +*/ func (*hashRolling) HashByte(b byte) uint32 { return uint32(b) + 1 } @@ -38,8 +42,10 @@ return uint32(factor*state + h.HashByte(add) - factor_remove*h.HashByte(rem)) } -/* Rolling hash for long distance long string matches. Stores one position - per bucket, bucket key is computed over a long region. */ +/* +Rolling hash for long distance long string matches. Stores one position +per bucket, bucket key is computed over a long region. +*/ type hashRolling struct { hasherCommon --- a/vendor/github.com/andybalholm/brotli/http.go +++ b/vendor/github.com/andybalholm/brotli/http.go @@ -46,24 +46,29 @@ bestQ := -1.0 specs := parseAccept(r.Header, "Accept-Encoding") for _, offer := range offers { + // A spec that names the offer takes precedence over "*", + // wherever it appears in the header. + q, wildcardQ := -1.0, -1.0 for _, spec := range specs { - if spec.Q > bestQ && strings.EqualFold(spec.Value, offer) { - bestQ = spec.Q - bestOffer = offer + if strings.EqualFold(spec.Value, offer) { + if spec.Q > q { + q = spec.Q + } + } else if spec.Value == "*" && spec.Q > wildcardQ { + wildcardQ = spec.Q } } + if q < 0 { + q = wildcardQ + } + if q > bestQ { + bestQ = q + bestOffer = offer + } } - switch bestQ { - case 0: + + if bestQ == 0 { bestOffer = "" - case -1.0: - // No specs matched, so we fall back to "*", if it is present. - for _, spec := range specs { - if spec.Value == "*" && spec.Q > bestQ { - bestQ = spec.Q - bestOffer = offers[0] - } - } } return bestOffer } --- a/vendor/github.com/andybalholm/brotli/huffman.go +++ b/vendor/github.com/andybalholm/brotli/huffman.go @@ -10,8 +10,10 @@ const huffmanMaxCodeLength = 15 -/* Maximum possible Huffman table size for an alphabet size of (index * 32), - max code length 15 and root table bits 8. */ +/* +Maximum possible Huffman table size for an alphabet size of (index * 32), +max code length 15 and root table bits 8. +*/ var kMaxHuffmanTableSize = []uint16{ 256, 402, @@ -363,9 +365,11 @@ const reverseBitsLowest = (uint64(1) << (reverseBitsMax - 1 + reverseBitsBase)) -/* Returns reverse(num >> BROTLI_REVERSE_BITS_BASE, BROTLI_REVERSE_BITS_MAX), - where reverse(value, len) is the bit-wise reversal of the len least - significant bits of value. */ +/* +Returns reverse(num >> BROTLI_REVERSE_BITS_BASE, BROTLI_REVERSE_BITS_MAX), +where reverse(value, len) is the bit-wise reversal of the len least +significant bits of value. +*/ func reverseBits8(num uint64) uint64 { return uint64(kReverseBits[num]) } @@ -382,9 +386,11 @@ } } -/* Returns the table width of the next 2nd level table. |count| is the histogram - of bit lengths for the remaining symbols, |len| is the code length of the - next processed symbol. */ +/* +Returns the table width of the next 2nd level table. |count| is the histogram +of bit lengths for the remaining symbols, |len| is the code length of the +next processed symbol. +*/ func nextTableBitSize(count []uint16, len int, root_bits int) int { var left int = 1 << uint(len-root_bits) for len < huffmanMaxCodeLength { --- a/vendor/github.com/andybalholm/brotli/matchfinder/m4.go +++ b/vendor/github.com/andybalholm/brotli/matchfinder/m4.go @@ -102,10 +102,31 @@ e.NextEmit = len(q.history) q.history = append(q.history, src...) if q.ChainLength > 0 { - q.chain = append(q.chain, make([]uint32, len(src))...) + oldLen := len(q.chain) + newLen := oldLen + len(src) + if cap(q.chain) < newLen { + newCap := 2 * cap(q.chain) + if newCap < newLen { + newCap = newLen + } + newChain := make([]uint32, newLen, newCap) + copy(newChain, q.chain) + q.chain = newChain + } else { + q.chain = q.chain[:newLen] + clear(q.chain[oldLen:]) + } } src = q.history + hashMask := (uint64(1) << (8 * q.HashLen)) - 1 + hashShift := uint(64 - q.TableBits) + table := q.table + chain := q.chain + chainLength := q.ChainLength + maxDistance := q.MaxDistance + minLength := q.MinLength + // matches stores the matches that have been found but not emitted, // in reverse order. (matches[0] is the most recent one.) var matches [3]absoluteMatch @@ -117,7 +138,7 @@ if matches[1].End > matches[0].Start { matches[1].End = matches[0].Start } - if matches[1].End-matches[1].Start >= q.MinLength && q.score(matches[1]) > 0 { + if matches[1].End-matches[1].Start >= minLength && q.score(matches[1]) > 0 { e.emit(matches[1]) } } @@ -128,53 +149,56 @@ // Look for a repeat match one byte after the current position. if matches[0] == (absoluteMatch{}) && len(e.Dst) > 0 { prevDistance := e.Dst[len(e.Dst)-1].Distance - if binary.LittleEndian.Uint32(src[i+1:]) == binary.LittleEndian.Uint32(src[i+1-prevDistance:]) && prevDistance != 0 { + repIndex := i + 1 - prevDistance + if 0 <= repIndex && repIndex < i+1 && + binary.LittleEndian.Uint32(src[i+1:]) == binary.LittleEndian.Uint32(src[repIndex:]) { // We have a 4-byte match. - m := extendMatch2(src, i+1, i+1-prevDistance, e.NextEmit+1) - if m.End-m.Start >= q.MinLength { + m := extendMatch2(src, i+1, repIndex, e.NextEmit+1) + if m.End-m.Start >= minLength { matches[0] = m } } } // Calculate and store the hash. - h := ((binary.LittleEndian.Uint64(src[i:]) & (1<<(8*q.HashLen) - 1)) * hashMul64) >> (64 - q.TableBits) - candidate := int(q.table[h]) - q.table[h] = uint32(i) - if q.ChainLength > 0 && candidate != 0 { - delta := i - candidate - q.chain[i] = uint32(delta) + cv := binary.LittleEndian.Uint64(src[i:]) + h := ((cv & hashMask) * hashMul64) >> hashShift + candidate := int(table[h]) + table[h] = uint32(i) + if chainLength > 0 && candidate != 0 { + chain[i] = uint32(i - candidate) } if i < matches[0].End && i != matches[0].End+2-q.HashLen { continue } - if candidate == 0 || i-candidate > q.MaxDistance { + if candidate == 0 || i-candidate > maxDistance { continue } // Look for a match. var currentMatch absoluteMatch + iVal := uint32(cv) - if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) { + if binary.LittleEndian.Uint32(src[candidate:]) == iVal { m := extendMatch2(src, i, candidate, e.NextEmit) - if m.End-m.Start > q.MinLength && q.score(m) > 0 { + if m.End-m.Start > minLength && q.score(m) > 0 { currentMatch = m } } - for j := 0; j < q.ChainLength; j++ { - delta := q.chain[candidate] + for j := 0; j < chainLength; j++ { + delta := chain[candidate] if delta == 0 { break } candidate -= int(delta) - if candidate <= 0 || i-candidate > q.MaxDistance { + if candidate <= 0 || i-candidate > maxDistance { break } - if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) { + if binary.LittleEndian.Uint32(src[candidate:]) == iVal { m := extendMatch2(src, i, candidate, e.NextEmit) - if m.End-m.Start > q.MinLength && q.score(m) > q.score(currentMatch) { + if m.End-m.Start > minLength && q.score(m) > q.score(currentMatch) { currentMatch = m } } @@ -295,9 +319,9 @@ case "amd64", "arm64": // As long as we are 8 or more bytes before the end of src, we can load and // compare 8 bytes at a time. If those 8 bytes are equal, repeat. - for j+8 < len(src) { - iBytes := binary.LittleEndian.Uint64(src[i:]) - jBytes := binary.LittleEndian.Uint64(src[j:]) + for j+8 <= len(src) { + iBytes := binary.LittleEndian.Uint64(src[i : i+8]) + jBytes := binary.LittleEndian.Uint64(src[j : j+8]) if iBytes != jBytes { // If those 8 bytes were not equal, XOR the two 8 byte values, and return // the index of the first byte that differs. The BSF instruction finds the @@ -309,9 +333,9 @@ } case "386": // On a 32-bit CPU, we do it 4 bytes at a time. - for j+4 < len(src) { - iBytes := binary.LittleEndian.Uint32(src[i:]) - jBytes := binary.LittleEndian.Uint32(src[j:]) + for j+4 <= len(src) { + iBytes := binary.LittleEndian.Uint32(src[i : i+4]) + jBytes := binary.LittleEndian.Uint32(src[j : j+4]) if iBytes != jBytes { return j + bits.TrailingZeros32(iBytes^jBytes)>>3 } --- a/vendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go +++ b/vendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go @@ -120,17 +120,35 @@ // Append src to the history buffer. historyLen := len(q.history) q.history = append(q.history, src...) - q.chain = append(q.chain, make([]uint32, len(src))...) + oldLen := len(q.chain) + newLen := oldLen + len(src) + if cap(q.chain) < newLen { + newCap := 2 * cap(q.chain) + if newCap < newLen { + newCap = newLen + } + newChain := make([]uint32, newLen, newCap) + copy(newChain, q.chain) + q.chain = newChain + } else { + q.chain = q.chain[:newLen] + clear(q.chain[oldLen:]) + } src = q.history + hashMask := (uint64(1) << (8 * q.HashLen)) - 1 + hashShift := uint(64 - q.TableBits) + table := q.table + chain := q.chain + // Calculate hashes and build the chain. for i := historyLen; i < len(src)-7; i++ { - h := ((binary.LittleEndian.Uint64(src[i:]) & (1<<(8*q.HashLen) - 1)) * hashMul64) >> (64 - q.TableBits) - candidate := int(q.table[h]) - q.table[h] = uint32(i) + cv := binary.LittleEndian.Uint64(src[i:]) + h := ((cv & hashMask) * hashMul64) >> hashShift + candidate := int(table[h]) + table[h] = uint32(i) if candidate != 0 { - delta := i - candidate - q.chain[i] = uint32(delta) + chain[i] = uint32(i - candidate) } } @@ -140,7 +158,7 @@ var prevMatch absoluteMatch i := historyLen for i < len(src)-7 { - delta := q.chain[i] + delta := chain[i] if delta == 0 { i++ continue @@ -165,7 +183,8 @@ } } - if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) { + iVal := binary.LittleEndian.Uint32(src[i:]) + if binary.LittleEndian.Uint32(src[candidate:]) == iVal { m := extendMatch2(src, i, candidate, max(historyLen, prevMatch.Start)) if m.End-m.Start > q.MinLength { currentMatch = m @@ -174,7 +193,7 @@ } for range q.ChainLength { - delta := q.chain[candidate] + delta := chain[candidate] if delta == 0 { break } @@ -182,7 +201,7 @@ if candidate <= 0 || i-candidate > q.MaxDistance { break } - if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) { + if binary.LittleEndian.Uint32(src[candidate:]) == iVal { m := extendMatch2(src, i, candidate, max(historyLen, prevMatch.Start)) if m.End-m.Start > q.MinLength && m.End-m.Start > currentMatch.End-currentMatch.Start { currentMatch = m --- a/vendor/github.com/andybalholm/brotli/metablock.go +++ b/vendor/github.com/andybalholm/brotli/metablock.go @@ -268,8 +268,11 @@ const maxStaticContexts = 13 -/* Greedy block splitter for one block category (literal, command or distance). - Gathers histograms for all context buckets. */ +/* +Greedy block splitter for one block category (literal, command or distance). + + Gathers histograms for all context buckets. +*/ type contextBlockSplitter struct { alphabet_size_ uint num_contexts_ uint @@ -328,10 +331,13 @@ self.last_histogram_ix_[0] = self.last_histogram_ix_[1] } -/* Does either of three things: - (1) emits the current block with a new block type; - (2) emits the current block with the type of the second last block; - (3) merges the current block with the last block. */ +/* +Does either of three things: + + (1) emits the current block with a new block type; + (2) emits the current block with the type of the second last block; + (3) merges the current block with the last block. +*/ func contextBlockSplitterFinishBlock(self *contextBlockSplitter, is_final bool) { var split *blockSplit = self.split_ var num_contexts uint = self.num_contexts_ @@ -459,8 +465,10 @@ } } -/* Adds the next symbol to the current block type and context. When the - current block reaches the target size, decides on merging the block. */ +/* +Adds the next symbol to the current block type and context. When the +current block reaches the target size, decides on merging the block. +*/ func contextBlockSplitterAddSymbol(self *contextBlockSplitter, symbol uint, context uint) { histogramAddLiteral(&self.histograms_[self.curr_histogram_ix_+context], symbol) self.block_size_++ --- a/vendor/github.com/andybalholm/brotli/metablock_command.go +++ b/vendor/github.com/andybalholm/brotli/metablock_command.go @@ -58,10 +58,13 @@ self.last_histogram_ix_[0] = self.last_histogram_ix_[1] } -/* Does either of three things: - (1) emits the current block with a new block type; - (2) emits the current block with the type of the second last block; - (3) merges the current block with the last block. */ +/* +Does either of three things: + + (1) emits the current block with a new block type; + (2) emits the current block with the type of the second last block; + (3) merges the current block with the last block. +*/ func blockSplitterFinishBlockCommand(self *blockSplitterCommand, is_final bool) { var split *blockSplit = self.split_ var last_entropy []float64 = self.last_entropy_[:] @@ -154,8 +157,11 @@ } } -/* Adds the next symbol to the current histogram. When the current histogram - reaches the target size, decides on merging the block. */ +/* +Adds the next symbol to the current histogram. When the current histogram + + reaches the target size, decides on merging the block. +*/ func blockSplitterAddSymbolCommand(self *blockSplitterCommand, symbol uint) { histogramAddCommand(&self.histograms_[self.curr_histogram_ix_], symbol) self.block_size_++ --- a/vendor/github.com/andybalholm/brotli/metablock_distance.go +++ b/vendor/github.com/andybalholm/brotli/metablock_distance.go @@ -58,10 +58,13 @@ self.last_histogram_ix_[0] = self.last_histogram_ix_[1] } -/* Does either of three things: - (1) emits the current block with a new block type; - (2) emits the current block with the type of the second last block; - (3) merges the current block with the last block. */ +/* +Does either of three things: + + (1) emits the current block with a new block type; + (2) emits the current block with the type of the second last block; + (3) merges the current block with the last block. +*/ func blockSplitterFinishBlockDistance(self *blockSplitterDistance, is_final bool) { var split *blockSplit = self.split_ var last_entropy []float64 = self.last_entropy_[:] @@ -154,8 +157,10 @@ } } -/* Adds the next symbol to the current histogram. When the current histogram - reaches the target size, decides on merging the block. */ +/* +Adds the next symbol to the current histogram. When the current histogram +reaches the target size, decides on merging the block. +*/ func blockSplitterAddSymbolDistance(self *blockSplitterDistance, symbol uint) { histogramAddDistance(&self.histograms_[self.curr_histogram_ix_], symbol) self.block_size_++ --- a/vendor/github.com/andybalholm/brotli/metablock_literal.go +++ b/vendor/github.com/andybalholm/brotli/metablock_literal.go @@ -58,10 +58,13 @@ self.last_histogram_ix_[0] = self.last_histogram_ix_[1] } -/* Does either of three things: - (1) emits the current block with a new block type; - (2) emits the current block with the type of the second last block; - (3) merges the current block with the last block. */ +/* +Does either of three things: + + (1) emits the current block with a new block type; + (2) emits the current block with the type of the second last block; + (3) merges the current block with the last block. +*/ func blockSplitterFinishBlockLiteral(self *blockSplitterLiteral, is_final bool) { var split *blockSplit = self.split_ var last_entropy []float64 = self.last_entropy_[:] @@ -154,8 +157,10 @@ } } -/* Adds the next symbol to the current histogram. When the current histogram - reaches the target size, decides on merging the block. */ +/* +Adds the next symbol to the current histogram. When the current histogram +reaches the target size, decides on merging the block. +*/ func blockSplitterAddSymbolLiteral(self *blockSplitterLiteral, symbol uint) { histogramAddLiteral(&self.histograms_[self.curr_histogram_ix_], symbol) self.block_size_++ --- a/vendor/github.com/andybalholm/brotli/prefix.go +++ b/vendor/github.com/andybalholm/brotli/prefix.go @@ -9,8 +9,10 @@ /* Functions for encoding of integers into prefix codes the amount of extra bits, and the actual values of the extra bits. */ -/* Here distance_code is an intermediate code, i.e. one of the special codes or - the actual distance increased by BROTLI_NUM_DISTANCE_SHORT_CODES - 1. */ +/* +Here distance_code is an intermediate code, i.e. one of the special codes or +the actual distance increased by BROTLI_NUM_DISTANCE_SHORT_CODES - 1. +*/ func prefixEncodeCopyDistance(distance_code uint, num_direct_codes uint, postfix_bits uint, code *uint16, extra_bits *uint32) { if distance_code < numDistanceShortCodes+num_direct_codes { *code = uint16(distance_code) --- a/vendor/github.com/andybalholm/brotli/quality.go +++ b/vendor/github.com/andybalholm/brotli/quality.go @@ -24,8 +24,10 @@ const minQualityForHqBlockSplitting = 10 -/* For quality below MIN_QUALITY_FOR_BLOCK_SPLIT there is no block splitting, - so we buffer at most this much literals and commands. */ +/* +For quality below MIN_QUALITY_FOR_BLOCK_SPLIT there is no block splitting, +so we buffer at most this much literals and commands. +*/ const maxNumDelayedSymbols = 0x2FFF /* Returns hash-table size for quality levels 0 and 1. */ @@ -102,11 +104,14 @@ return lgblock } -/* Returns log2 of the size of main ring buffer area. - Allocate at least lgwin + 1 bits for the ring buffer so that the newly - added block fits there completely and we still get lgwin bits and at least - read_block_size_bits + 1 bits because the copy tail length needs to be - smaller than ring-buffer size. */ +/* +Returns log2 of the size of main ring buffer area. + + Allocate at least lgwin + 1 bits for the ring buffer so that the newly + added block fits there completely and we still get lgwin bits and at least + read_block_size_bits + 1 bits because the copy tail length needs to be + smaller than ring-buffer size. +*/ func computeRbBits(params *encoderParams) int { return 1 + brotli_max_int(int(params.lgwin), params.lgblock) } @@ -116,12 +121,14 @@ return uint(1) << uint(bits) } -/* When searching for backward references and have not seen matches for a long - time, we can skip some match lookups. Unsuccessful match lookups are very - expensive and this kind of a heuristic speeds up compression quite a lot. - At first 8 byte strides are taken and every second byte is put to hasher. - After 4x more literals stride by 16 bytes, every put 4-th byte to hasher. - Applied only to qualities 2 to 9. */ +/* +When searching for backward references and have not seen matches for a long +time, we can skip some match lookups. Unsuccessful match lookups are very +expensive and this kind of a heuristic speeds up compression quite a lot. +At first 8 byte strides are taken and every second byte is put to hasher. +After 4x more literals stride by 16 bytes, every put 4-th byte to hasher. +Applied only to qualities 2 to 9. +*/ func literalSpreeLengthForSparseSearch(params *encoderParams) uint { if params.quality < 9 { return 64 --- a/vendor/github.com/andybalholm/brotli/ringbuffer.go +++ b/vendor/github.com/andybalholm/brotli/ringbuffer.go @@ -6,15 +6,22 @@ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* A ringBuffer(window_bits, tail_bits) contains `1 << window_bits' bytes of - data in a circular manner: writing a byte writes it to: - `position() % (1 << window_bits)'. - For convenience, the ringBuffer array contains another copy of the - first `1 << tail_bits' bytes: - buffer_[i] == buffer_[i + (1 << window_bits)], if i < (1 << tail_bits), - and another copy of the last two bytes: - buffer_[-1] == buffer_[(1 << window_bits) - 1] and - buffer_[-2] == buffer_[(1 << window_bits) - 2]. */ +/* +A ringBuffer(window_bits, tail_bits) contains `1 << window_bits' bytes of +data in a circular manner: writing a byte writes it to: + + `position() % (1 << window_bits)'. + +For convenience, the ringBuffer array contains another copy of the +first `1 << tail_bits' bytes: + + buffer_[i] == buffer_[i + (1 << window_bits)], if i < (1 << tail_bits), + +and another copy of the last two bytes: + + buffer_[-1] == buffer_[(1 << window_bits) - 1] and + buffer_[-2] == buffer_[(1 << window_bits) - 2]. +*/ type ringBuffer struct { size_ uint32 mask_ uint32 @@ -41,8 +48,10 @@ const kSlackForEightByteHashingEverywhere uint = 7 -/* Allocates or re-allocates data_ to the given length + plus some slack - region before and after. Fills the slack regions with zeros. */ +/* +Allocates or re-allocates data_ to the given length + plus some slack +region before and after. Fills the slack regions with zeros. +*/ func ringBufferInitBuffer(buflen uint32, rb *ringBuffer) { var new_data []byte var i uint --- a/vendor/github.com/andybalholm/brotli/static_dict.go +++ b/vendor/github.com/andybalholm/brotli/static_dict.go @@ -14,10 +14,11 @@ const kInvalidMatch uint32 = 0xFFFFFFF -/* Copyright 2013 Google Inc. All Rights Reserved. +/* +Copyright 2013 Google Inc. All Rights Reserved. - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ func hash(data []byte) uint32 { var h uint32 = binary.LittleEndian.Uint32(data) * kDictHashMul32 --- a/vendor/github.com/andybalholm/brotli/utf8_util.go +++ b/vendor/github.com/andybalholm/brotli/utf8_util.go @@ -10,9 +10,11 @@ const kMinUTF8Ratio float64 = 0.75 -/* Returns 1 if at least min_fraction of the bytes between pos and - pos + length in the (data, mask) ring-buffer is UTF8-encoded, otherwise - returns 0. */ +/* +Returns 1 if at least min_fraction of the bytes between pos and +pos + length in the (data, mask) ring-buffer is UTF8-encoded, otherwise +returns 0. +*/ func parseAsUTF8(symbol *int, input []byte, size uint) uint { /* ASCII */ if input[0]&0x80 == 0 { --- a/vendor/github.com/andybalholm/brotli/write_bits.go +++ b/vendor/github.com/andybalholm/brotli/write_bits.go @@ -10,21 +10,23 @@ /* Write bits into a byte array. */ -/* This function writes bits into bytes in increasing addresses, and within - a byte least-significant-bit first. +/* +This function writes bits into bytes in increasing addresses, and within +a byte least-significant-bit first. - The function can write up to 56 bits in one go with WriteBits - Example: let's assume that 3 bits (Rs below) have been written already: +The function can write up to 56 bits in one go with WriteBits +Example: let's assume that 3 bits (Rs below) have been written already: - BYTE-0 BYTE+1 BYTE+2 +BYTE-0 BYTE+1 BYTE+2 - 0000 0RRR 0000 0000 0000 0000 +0000 0RRR 0000 0000 0000 0000 - Now, we could write 5 or less bits in MSB by just sifting by 3 - and OR'ing to BYTE-0. +Now, we could write 5 or less bits in MSB by just sifting by 3 +and OR'ing to BYTE-0. - For n bits, we take the last 5 bits, OR that with high bits in BYTE-0, - and locate the rest in BYTE+1, BYTE+2, etc. */ +For n bits, we take the last 5 bits, OR that with high bits in BYTE-0, +and locate the rest in BYTE+1, BYTE+2, etc. +*/ func writeBits(n_bits uint, bits uint64, pos *uint, array []byte) { /* This branch of the code can write up to 56 bits at a time, 7 bits are lost by being perhaps already in *p and at least --- a/vendor/github.com/apache/thrift/LICENSE +++ b/vendor/github.com/apache/thrift/LICENSE @@ -210,13 +210,6 @@ conditions of the following licenses. -------------------------------------------------- -Portions of the following files are licensed under the MIT License: - - lib/erl/src/Makefile.am - -Please see doc/otp-base-license.txt for the full terms of this license. - --------------------------------------------------- For the aclocal/ax_boost_base.m4 and contrib/fb303/aclocal/ax_boost_base.m4 components: # Copyright (c) 2007 Thomas Porschberg --- a/vendor/github.com/apache/thrift/lib/go/thrift/binary_protocol.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/binary_protocol.go @@ -293,6 +293,12 @@ if p.cfg.GetTBinaryStrictRead() { return name, typeId, seqId, NewTProtocolExceptionWithType(BAD_VERSION, fmt.Errorf("Missing version in ReadMessageBegin")) } + // Without a version, size is the length of the name, and it is held to + // the same limit as the length of any other string. + e = checkSizeForProtocol(size, p.cfg) + if e != nil { + return name, typeId, seqId, e + } name, e2 := p.readStringBody(size) if e2 != nil { return name, typeId, seqId, e2 @@ -356,7 +362,7 @@ return } minElemSize := p.getMinSerializedSize(kType) + p.getMinSerializedSize(vType) - err = checkContainerSizeForProtocol(int64(size32), minElemSize, p.cfg) + err = checkContainerSizeForProtocol(int64(size32), minElemSize, p.trans.RemainingBytes(), p.cfg) if err != nil { return } @@ -381,7 +387,7 @@ return } minElemSize := p.getMinSerializedSize(elemType) - err = checkContainerSizeForProtocol(int64(size32), minElemSize, p.cfg) + err = checkContainerSizeForProtocol(int64(size32), minElemSize, p.trans.RemainingBytes(), p.cfg) if err != nil { return } @@ -407,7 +413,7 @@ return } minElemSize := p.getMinSerializedSize(elemType) - err = checkContainerSizeForProtocol(int64(size32), minElemSize, p.cfg) + err = checkContainerSizeForProtocol(int64(size32), minElemSize, p.trans.RemainingBytes(), p.cfg) if err != nil { return } @@ -590,18 +596,35 @@ // It tries to read size bytes from trans, in a way that prevents large // allocations when size is insanely large (mostly caused by malformed message), // or smaller than bytes.MinRead. +// +// Above bytes.MinRead the buffer starts small and doubles as the data arrives, +// so a size the sender made up costs no more than the bytes it actually sends, +// and the growth stops at size, so a well-formed message ends up in a buffer +// holding exactly what it asked for. func safeReadBytes(size int32, trans io.Reader) ([]byte, error) { if size < 0 { return nil, nil } - if size > bytes.MinRead { - // Use bytes.Buffer to prevent allocating size bytes when size is very large - buf := new(bytes.Buffer) - _, err := io.CopyN(buf, trans, int64(size)) - return buf.Bytes(), err - } - // Allocate size bytes - b := make([]byte, size) - n, err := io.ReadFull(trans, b) - return b[:n], err + if size <= bytes.MinRead { + // Allocate size bytes + b := make([]byte, size) + n, err := io.ReadFull(trans, b) + return b[:n], err + } + + want := int(size) + buf := make([]byte, 0, bytes.MinRead) + for len(buf) < want { + if len(buf) == cap(buf) { + grown := make([]byte, len(buf), min(cap(buf)*2, want)) + copy(grown, buf) + buf = grown + } + n, err := trans.Read(buf[len(buf):cap(buf)]) + buf = buf[:len(buf)+n] + if err != nil { + return buf, err + } + } + return buf, nil } --- a/vendor/github.com/apache/thrift/lib/go/thrift/buffered_transport.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/buffered_transport.go @@ -87,8 +87,11 @@ return p.tp.Flush(ctx) } +// RemainingBytes returns the bytes still buffered here plus whatever the +// underlying transport still has. The bytes already pulled into the read buffer +// have to be added back, because the underlying transport no longer counts them. func (p *TBufferedTransport) RemainingBytes() (num_bytes uint64) { - return p.tp.RemainingBytes() + return addRemainingBytes(p.tp.RemainingBytes(), uint64(p.Reader.Buffered())) } // SetTConfiguration implements TConfigurationSetter for propagation. --- a/vendor/github.com/apache/thrift/lib/go/thrift/compact_protocol.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/compact_protocol.go @@ -499,7 +499,7 @@ valueType, _ = p.getTType(tCompactType(keyAndValueType & 0xf)) minElemSize := p.getMinSerializedSize(keyType) + p.getMinSerializedSize(valueType) - err = checkContainerSizeForProtocol(int64(size32), minElemSize, p.cfg) + err = checkContainerSizeForProtocol(int64(size32), minElemSize, p.trans.RemainingBytes(), p.cfg) if err != nil { return } @@ -533,7 +533,7 @@ } minElemSize := p.getMinSerializedSize(elemType) - err = checkContainerSizeForProtocol(int64(size), minElemSize, p.cfg) + err = checkContainerSizeForProtocol(int64(size), minElemSize, p.trans.RemainingBytes(), p.cfg) if err != nil { return } --- a/vendor/github.com/apache/thrift/lib/go/thrift/configuration.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/configuration.go @@ -110,6 +110,15 @@ // MaxMessageSize will be used instead. MaxFrameSize int32 + // The maximum number of elements in a single list, set or map read by a + // TProtocol. THeaderTransport holds the transform count of a frame header + // to it as well. + // + // If <= 0, the element count is not limited on its own. It is still + // bounded by MaxMessageSize and, where the transport knows it, by the + // number of bytes left to read. + MaxContainerSize int32 + // Connect and socket timeouts to be used by TSocket and TSSLSocket. // // 0 means no timeout. @@ -174,6 +183,18 @@ return maxFrameSize } +// GetMaxContainerSize returns the max number of elements in a single list, set +// or map an implementation should follow. +// +// It's nil-safe. math.MaxInt32, which leaves the element count without a limit +// of its own, will be returned if tc is nil or tc.MaxContainerSize is <= 0. +func (tc *TConfiguration) GetMaxContainerSize() int32 { + if tc == nil || tc.MaxContainerSize <= 0 { + return math.MaxInt32 + } + return tc.MaxContainerSize +} + // GetConnectTimeout returns the connect timeout should be used by TSocket and // TSSLSocket. // @@ -337,8 +358,15 @@ // container with the given wire-supplied element count, where each element // occupies at least minElemSize bytes. The count is range-checked and the // product is computed in 64-bit arithmetic, so the value handed to -// checkSizeForProtocol always stays within int32 range. -func checkContainerSizeForProtocol(size int64, minElemSize int32, cfg *TConfiguration) error { +// checkSizeForProtocol always stays within int32 range. A count above +// cfg.GetMaxContainerSize() is rejected as well. +// +// remaining is the transport's ReadSizeProvider.RemainingBytes value. When the +// transport knows how much is left, a count whose elements cannot fit in that +// many bytes is rejected here, before the caller uses it to size an allocation. +// This mirrors the checkReadBytesAvailable step the Java and C++ protocols +// perform on the same three headers. +func checkContainerSizeForProtocol(size int64, minElemSize int32, remaining uint64, cfg *TConfiguration) error { if size < 0 { return NewTProtocolExceptionWithType( NEGATIVE_SIZE, @@ -351,6 +379,12 @@ fmt.Errorf("size exceeded max allowed: %d", size), ) } + if limit := cfg.GetMaxContainerSize(); size > int64(limit) { + return NewTProtocolExceptionWithType( + SIZE_LIMIT, + fmt.Errorf("container of %d elements exceeds the limit of %d", size, limit), + ) + } totalMinSize := size * int64(minElemSize) if totalMinSize > math.MaxInt32 { return NewTProtocolExceptionWithType( @@ -358,6 +392,15 @@ fmt.Errorf("size exceeded max allowed: %d", totalMinSize), ) } + if remaining != UnknownRemainingBytes && uint64(totalMinSize) > remaining { + return NewTProtocolExceptionWithType( + SIZE_LIMIT, + fmt.Errorf( + "container of %d elements needs at least %d bytes, only %d remaining", + size, totalMinSize, remaining, + ), + ) + } return checkSizeForProtocol(int32(totalMinSize), cfg) } --- a/vendor/github.com/apache/thrift/lib/go/thrift/exception.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/exception.go @@ -35,8 +35,7 @@ func PrependError(prepend string, err error) error { msg := prepend + err.Error() - var te TException - if errors.As(err, &te) { + if te, ok := errors.AsType[TException](err); ok { switch te.TExceptionType() { case TExceptionTypeTransport: if t, ok := err.(TTransportException); ok { @@ -47,8 +46,7 @@ return prependTProtocolException(prepend, t) } case TExceptionTypeApplication: - var t TApplicationException - if errors.As(err, &t) { + if t, ok := errors.AsType[TApplicationException](err); ok { return NewTApplicationException(t.TypeId(), msg) } } --- a/vendor/github.com/apache/thrift/lib/go/thrift/framed_transport.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/framed_transport.go @@ -140,30 +140,32 @@ } }() - if p.readBuf != nil { - - read, err = p.readBuf.Read(buf) - if err != io.EOF { - return + // Loop rather than recurse: an empty frame yields no bytes, so a peer + // can drive one iteration per 4 bytes it sends, and a stack frame per + // iteration would not be bounded by anything. + for { + if p.readBuf != nil { + read, err = p.readBuf.Read(buf) + if err != io.EOF { + return + } + + // For bytes.Buffer.Read, EOF would only happen when read is zero, + // but still, do a sanity check, + // in case that behavior is changed in a future version of go stdlib. + // When that happens, just return nil error, + // and let the caller call Read again to read the next frame. + if read > 0 { + return read, nil + } } - // For bytes.Buffer.Read, EOF would only happen when read is zero, - // but still, do a sanity check, - // in case that behavior is changed in a future version of go stdlib. - // When that happens, just return nil error, - // and let the caller call Read again to read the next frame. - if read > 0 { - return read, nil + // Reaching here means that the last Read finished the last frame, + // so we need to read the next frame into readBuf now. + if err = p.readFrame(); err != nil { + return read, err } } - - // Reaching here means that the last Read finished the last frame, - // so we need to read the next frame into readBuf now. - if err = p.readFrame(); err != nil { - return read, err - } - newRead, err := p.Read(buf[read:]) - return read + newRead, err } func (p *TFramedTransport) ReadByte() (c byte, err error) { @@ -200,11 +202,15 @@ func (p *TFramedTransport) Flush(ctx context.Context) error { size := p.writeBuf.Len() - if uint64(size) > uint64(math.MaxUint32) { - return NewTTransportException(UNKNOWN_TRANSPORT_EXCEPTION, fmt.Sprintf("frame too large: %d bytes exceeds uint32 max", size)) - } - defer bufPool.put(&p.writeBuf) + + // readFrame refuses a frame larger than the configured maximum, and so does + // a peer holding the same configuration. THeaderTransport.Flush holds the + // frames it writes to the same limit. A refused frame is dropped with the + // buffer. + if err := checkWriteFrameSize(p.cfg, size); err != nil { + return err + } buf := p.buffer[:4] binary.BigEndian.PutUint32(buf, uint32(size)) _, err := p.transport.Write(buf) --- a/vendor/github.com/apache/thrift/lib/go/thrift/header_transport.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/header_transport.go @@ -28,6 +28,7 @@ "errors" "fmt" "io" + "math" "slices" ) @@ -143,8 +144,8 @@ // stops at and returns the first error encountered. func (tr *TransformReader) Close() error { // Call closers in reversed order - for i := len(tr.closers) - 1; i >= 0; i-- { - if err := tr.closers[i].Close(); err != nil { + for _, v := range slices.Backward(tr.closers) { + if err := v.Close(); err != nil { return err } } @@ -177,6 +178,38 @@ return nil } +// limitedTransformReader reads the output of a read transform, and fails the +// read that would take that output past limit bytes. +type limitedTransformReader struct { + r io.Reader + limit int64 + read int64 + err error +} + +func (l *limitedTransformReader) Read(p []byte) (int, error) { + if l.err != nil { + return 0, l.err + } + // Read at most one byte past the limit: output that ends right at the + // limit is read to its end, and the extra byte shows output that goes on. + if left := l.limit - l.read; int64(len(p)) > left+1 { + p = p[:left+1] + } + n, err := l.r.Read(p) + if l.read+int64(n) > l.limit { + l.err = NewTProtocolExceptionWithType( + SIZE_LIMIT, + fmt.Errorf("frame too large after transform: more than %d bytes", l.limit), + ) + n = int(l.limit - l.read) + l.read = l.limit + return n, l.err + } + l.read += int64(n) + return n, err +} + // TransformWriter is an io.WriteCloser that handles transforms writing. type TransformWriter struct { io.Writer @@ -204,8 +237,8 @@ // stops at and returns the first error encountered. func (tw *TransformWriter) Close() error { // Call closers in reversed order - for i := len(tw.closers) - 1; i >= 0; i-- { - if err := tw.closers[i].Close(); err != nil { + for _, v := range slices.Backward(tw.closers) { + if err := v.Close(); err != nil { return err } } @@ -376,6 +409,17 @@ errors.New("frame too large"), ) } + // A frame has to carry at least the 32-bit word peeked at further down, + // which is what says whether it holds headers or a bare message. Slicing + // to size32 on a shorter frame reads past what was copied into the frame + // buffer, and that buffer comes from a pool, so what it reads is whatever + // the previous user of the buffer left behind. + if frameSize < size32 { + return NewTProtocolExceptionWithType( + INVALID_DATA, + fmt.Errorf("frame too small: %d bytes", frameSize), + ) + } t.reader.Discard(size32) // Read the frame fully into frameBuffer. @@ -469,6 +513,15 @@ if err != nil { return err } + // transformCount is an element count read straight off the wire, not a + // buffer length, so the frame-size and header-length checks above do not + // bound it. Validate it the same way every other wire-supplied container + // count (list/set/map sizes) is validated, before it sizes any allocation. + if err = checkContainerSizeForProtocol( + int64(transformCount), 1, headerBuf.RemainingBytes(), t.cfg, + ); err != nil { + return err + } if transformCount > 0 { reader := NewTransformReaderWithCapacity( t.frameBuffer, @@ -499,6 +552,15 @@ if err := reader.AddTransform(id); err != nil { return err } + if id == TransformZlib { + // ReadFrame held the frame to the maximum frame + // size as it came off the wire. Hold what each + // inflate makes of it to the same size. + reader.Reader = &limitedTransformReader{ + r: reader.Reader, + limit: int64(t.cfg.GetMaxFrameSize()), + } + } } } @@ -558,11 +620,18 @@ // Then, 99% of the case when calling this Read frame is already read // into frameReader. ReadFrame here is more of preventing bugs that // didn't call ReadFrame before calling Read. - err = t.ReadFrame(context.Background()) - if err != nil { - return - } - if t.frameReader != nil { + // Loop rather than recurse: a frame whose header block fills it yields no + // payload bytes, so a peer can drive one iteration per 18 bytes it sends, + // and a stack frame per iteration would not be bounded by anything. + for { + err = t.ReadFrame(context.Background()) + if err != nil { + return + } + if t.frameReader == nil { + return t.reader.Read(p) + } + read, err = t.frameReader.Read(p) if err == nil && t.frameBuffer.Len() <= 0 { // the last Read finished the frame, do endOfFrame @@ -583,12 +652,11 @@ // as otherwise we would return 0 and nil, // which is a case not handled well by most // protocol implementations. - return t.Read(p) + continue } } return } - return t.reader.Read(p) } // Write writes data to the write buffer. @@ -601,6 +669,21 @@ return t.writeBuffer.Write(p) } +// checkWriteFrameSize refuses a frame of size bytes that a reader holding cfg +// would refuse. THeaderMaxFrameSize is below the largest length the frame's +// 32-bit length word can carry, so a frame that passes is written with its +// true length. TFramedTransport.Flush writes the same length word and applies +// the same check. +func checkWriteFrameSize(cfg *TConfiguration, size int) error { + if uint64(size) > uint64(THeaderMaxFrameSize) || int64(size) > int64(cfg.GetMaxFrameSize()) { + return NewTProtocolExceptionWithType( + SIZE_LIMIT, + fmt.Errorf("frame too large: %d bytes", size), + ) + } + return nil +} + // Flush writes the appropriate header and the write buffer to the underlying transport. func (t *THeaderTransport) Flush(ctx context.Context) error { if t.writeBuffer == nil || t.writeBuffer.Len() == 0 { @@ -657,12 +740,21 @@ } } + // The header length is a count of 4-byte words, carried in 16 bits. + headerWords := headers.Len() / 4 + if headerWords > math.MaxUint16 { + return NewTProtocolExceptionWithType( + SIZE_LIMIT, + fmt.Errorf("headers too large: %d bytes", headers.Len()), + ) + } + payload := bufPool.get() defer bufPool.put(&payload) meta := headerMeta{ MagicFlags: THeaderHeaderMagic + t.Flags&THeaderFlagsMask, SequenceID: t.SequenceID, - HeaderLength: uint16(headers.Len() / 4), + HeaderLength: uint16(headerWords), } if err := binary.Write(payload, binary.BigEndian, meta); err != nil { return NewTTransportExceptionFromError(err) @@ -683,6 +775,9 @@ } // First write frame length + if err := checkWriteFrameSize(t.cfg, payload.Len()); err != nil { + return err + } buf := t.buffer[:size32] binary.BigEndian.PutUint32(buf, uint32(payload.Len())) if _, err := t.transport.Write(buf); err != nil { @@ -694,6 +789,9 @@ } case clientFramedBinary, clientFramedCompact: + if err := checkWriteFrameSize(t.cfg, t.writeBuffer.Len()); err != nil { + return err + } buf := t.buffer[:size32] binary.BigEndian.PutUint32(buf, uint32(t.writeBuffer.Len())) if _, err := t.transport.Write(buf); err != nil { @@ -723,15 +821,24 @@ return t.transport.Close() } -// RemainingBytes calls underlying transport's RemainingBytes. +// RemainingBytes returns the number of bytes left in the current frame. // -// Even in framed cases, because of all the possible compression transforms -// involved, the remaining frame size is likely to be different from the actual -// remaining readable bytes, so we don't bother to keep tracking the remaining -// frame size by ourselves and just use the underlying transport's -// RemainingBytes directly. +// The underlying transport's value cannot be used here: the frame has already +// been copied out of it into frameBuffer, so it no longer accounts for the +// bytes this transport can still deliver. +// +// When a compression transform is in play the frame buffer holds transformed +// bytes, which say nothing about how many bytes the transform will produce, so +// the answer is UnknownRemainingBytes. The same applies outside a frame, where +// reads come from a bufio.Reader over the underlying transport. func (t *THeaderTransport) RemainingBytes() uint64 { - return t.transport.RemainingBytes() + if t.frameReader == nil || t.frameBuffer == nil { + return UnknownRemainingBytes + } + if _, transformed := t.frameReader.(*TransformReader); transformed { + return UnknownRemainingBytes + } + return uint64(t.frameBuffer.Len()) } // GetReadHeaders returns the THeaderMap read from transport. @@ -821,6 +928,7 @@ // NewTHeaderTransportFactoryConf creates a new *THeaderTransportFactory with // the given *TConfiguration. func NewTHeaderTransportFactoryConf(factory TTransportFactory, conf *TConfiguration) TTransportFactory { + PropagateTConfiguration(factory, conf) return &THeaderTransportFactory{ Factory: factory, @@ -831,6 +939,7 @@ // GetTransport implements TTransportFactory. func (f *THeaderTransportFactory) GetTransport(trans TTransport) (TTransport, error) { if f.Factory != nil { + PropagateTConfiguration(trans, f.cfg) t, err := f.Factory.GetTransport(trans) if err != nil { return nil, err @@ -842,7 +951,7 @@ // SetTConfiguration implements TConfigurationSetter. func (f *THeaderTransportFactory) SetTConfiguration(cfg *TConfiguration) { - PropagateTConfiguration(f.Factory, f.cfg) + PropagateTConfiguration(f.Factory, cfg) f.cfg = cfg } --- a/vendor/github.com/apache/thrift/lib/go/thrift/http_client.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/http_client.go @@ -230,8 +230,7 @@ return uint64(len) } - const maxSize = ^uint64(0) - return maxSize // the truth is, we just don't know unless framed is used + return UnknownRemainingBytes } // Deprecated: Use NewTHttpClientTransportFactory instead. --- a/vendor/github.com/apache/thrift/lib/go/thrift/http_transport.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/http_transport.go @@ -34,6 +34,10 @@ w.Header().Add("Content-Type", "application/x-thrift") transport := NewStreamTransport(r.Body, w) + // A request that declares its length tells the decoder how much it can + // possibly read; without this the transport reports unknown and the + // container element-count check has nothing to bound against. + transport.setKnownSize(r.ContentLength) processor.Process(r.Context(), inPfactory.GetProtocol(transport), outPfactory.GetProtocol(transport)) }) } --- a/vendor/github.com/apache/thrift/lib/go/thrift/iostream_transport.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/iostream_transport.go @@ -31,6 +31,22 @@ io.Writer isReadWriter bool closed bool + + // knownSize is the number of bytes the caller knows the reader holds, or + // zero when it does not know. See setKnownSize. + knownSize int64 +} + +// setKnownSize records how many bytes the reader is known to hold, for callers +// that are told the length up front - an HTTP request with a Content-Length, +// for example. A value of zero or less means unknown and is ignored. +// +// The recorded value is an upper bound and is deliberately not decremented as +// bytes are consumed: over-reporting only makes the container check accept a +// count it could have rejected, while under-reporting would reject a message +// that is in fact complete. +func (p *StreamTransport) setKnownSize(size int64) { + p.knownSize = size } type StreamTransportFactory struct { @@ -209,8 +225,10 @@ } func (p *StreamTransport) RemainingBytes() (num_bytes uint64) { - const maxSize = ^uint64(0) - return maxSize // the truth is, we just don't know unless framed is used + if p.knownSize > 0 { + return uint64(p.knownSize) + } + return UnknownRemainingBytes } // SetTConfiguration implements TConfigurationSetter for propagation. --- a/vendor/github.com/apache/thrift/lib/go/thrift/json_protocol.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/json_protocol.go @@ -317,7 +317,7 @@ size = int(iSize) minElemSize := p.getMinSerializedSize(keyType) + p.getMinSerializedSize(valueType) - err = checkContainerSizeForProtocol(iSize, minElemSize, p.cfg) + err = checkContainerSizeForProtocol(iSize, minElemSize, p.RemainingBytes(), p.cfg) if err != nil { return keyType, valueType, 0, err } @@ -497,7 +497,7 @@ size = int(nSize) minElemSize := p.getMinSerializedSize(elemType) - err = checkContainerSizeForProtocol(nSize, minElemSize, p.cfg) + err = checkContainerSizeForProtocol(nSize, minElemSize, p.RemainingBytes(), p.cfg) if err != nil { return elemType, 0, err } @@ -569,39 +569,5 @@ } // Return the minimum number of bytes a type will consume on the wire -func (p *TJSONProtocol) getMinSerializedSize(ttype TType) int32 { - switch ttype { - case STOP: - return 1 // T_STOP needs to count itself - case VOID: - return 1 // T_VOID needs to count itself - case BOOL: - return 1 // written as int - case BYTE: - return 1 - case DOUBLE: - return 1 - case I16: - return 1 - case I32: - return 1 - case I64: - return 1 - case STRING: - return 2 // empty string - case STRUCT: - return 2 // empty struct - case MAP: - return 2 // empty map - case SET: - return 2 // empty set - case LIST: - return 2 // empty list - case UUID: - return 16 // empty UUID - default: - return 1 // unknown type - } -} var _ TConfigurationSetter = (*TJSONProtocol)(nil) --- a/vendor/github.com/apache/thrift/lib/go/thrift/logger.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/logger.go @@ -22,7 +22,6 @@ import ( "log" "os" - "testing" ) // Logger is a simple wrapper of a logging function. @@ -65,9 +64,15 @@ // // It fails the test when being called. // +// The argument is declared as the one method the implementation calls rather +// than as testing.TB, so that the library itself does not import testing. +// A *testing.T, a *testing.B, or a testing.TB still satisfies it. +// // Deprecated: This is no longer used by any thrift go library code, // will be removed in the future version. -func TestLogger(tb testing.TB) Logger { +func TestLogger(tb interface { + Errorf(format string, args ...any) +}) Logger { return func(msg string) { tb.Errorf("logger called with msg: %q", msg) } --- a/vendor/github.com/apache/thrift/lib/go/thrift/middleware.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/middleware.go @@ -21,6 +21,7 @@ import ( "context" + "slices" ) // ProcessorMiddleware is a function that can be passed to WrapProcessor to wrap the @@ -40,8 +41,8 @@ for name, processorFunc := range processor.ProcessorMap() { wrapped := processorFunc // Add middlewares in reverse so the first in the list is the outermost. - for i := len(middlewares) - 1; i >= 0; i-- { - wrapped = middlewares[i](name, wrapped) + for _, middleware := range slices.Backward(middlewares) { + wrapped = middleware(name, wrapped) } processor.AddToProcessorMap(name, wrapped) } @@ -98,8 +99,8 @@ // Middlewares[0] -> Middlewares[1] -> ... -> Middlewares[n] func WrapClient(client TClient, middlewares ...ClientMiddleware) TClient { // Add middlewares in reverse so the first in the list is the outermost. - for i := len(middlewares) - 1; i >= 0; i-- { - client = middlewares[i](client) + for _, middleware := range slices.Backward(middlewares) { + client = middleware(client) } return client } --- a/vendor/github.com/apache/thrift/lib/go/thrift/pointerize.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/pointerize.go @@ -21,8 +21,10 @@ // Pointer is the generic (type parameter) version of the helper function that // converts types to pointer types. +// +//go:fix inline func Pointer[T any](v T) *T { - return &v + return new(v) } /////////////////////////////////////////////////////////////////////////////// @@ -44,16 +46,41 @@ // exported thrift structs, we factor such helpers here. /////////////////////////////////////////////////////////////////////////////// -func Float32Ptr(v float32) *float32 { return &v } -func Float64Ptr(v float64) *float64 { return &v } -func IntPtr(v int) *int { return &v } -func Int8Ptr(v int8) *int8 { return &v } -func Int16Ptr(v int16) *int16 { return &v } -func Int32Ptr(v int32) *int32 { return &v } -func Int64Ptr(v int64) *int64 { return &v } -func StringPtr(v string) *string { return &v } -func Uint32Ptr(v uint32) *uint32 { return &v } -func Uint64Ptr(v uint64) *uint64 { return &v } -func BoolPtr(v bool) *bool { return &v } -func ByteSlicePtr(v []byte) *[]byte { return &v } -func TuuidPtr(v Tuuid) *Tuuid { return &v } +//go:fix inline +func Float32Ptr(v float32) *float32 { return new(v) } + +//go:fix inline +func Float64Ptr(v float64) *float64 { return new(v) } + +//go:fix inline +func IntPtr(v int) *int { return new(v) } + +//go:fix inline +func Int8Ptr(v int8) *int8 { return new(v) } + +//go:fix inline +func Int16Ptr(v int16) *int16 { return new(v) } + +//go:fix inline +func Int32Ptr(v int32) *int32 { return new(v) } + +//go:fix inline +func Int64Ptr(v int64) *int64 { return new(v) } + +//go:fix inline +func StringPtr(v string) *string { return new(v) } + +//go:fix inline +func Uint32Ptr(v uint32) *uint32 { return new(v) } + +//go:fix inline +func Uint64Ptr(v uint64) *uint64 { return new(v) } + +//go:fix inline +func BoolPtr(v bool) *bool { return new(v) } + +//go:fix inline +func ByteSlicePtr(v []byte) *[]byte { return new(v) } + +//go:fix inline +func TuuidPtr(v Tuuid) *Tuuid { return new(v) } --- a/vendor/github.com/apache/thrift/lib/go/thrift/protocol.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/protocol.go @@ -186,3 +186,20 @@ return NewTProtocolExceptionWithType(INVALID_DATA, fmt.Errorf("Unknown data type %d", fieldType)) } } + +// maxPreallocSize is the largest number of elements a wire-supplied container +// count may reserve up front. Element counts are read before any element is, +// so a peer can name a count it never backs with data; capping what the count +// reserves keeps the cost of that proportional to the bytes actually sent, +// while still reserving the exact capacity for the containers that fit under +// the cap. Larger containers grow by append, which costs a few reallocations. +const maxPreallocSize = 1024 + +// PreallocSize returns the capacity to reserve for a container whose declared +// element count is size. It is used by generated code. +func PreallocSize(size int) int { + if size < 0 { + return 0 + } + return min(size, maxPreallocSize) +} --- a/vendor/github.com/apache/thrift/lib/go/thrift/simple_json_protocol.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/simple_json_protocol.go @@ -113,6 +113,14 @@ }) } +// RemainingBytes implements ReadSizeProvider. +// +// The protocol reads through a bufio.Reader, so the bytes already pulled out of +// the transport have to be added back to what the transport still reports. +func (p *TSimpleJSONProtocol) RemainingBytes() uint64 { + return addRemainingBytes(p.trans.RemainingBytes(), uint64(p.reader.Buffered())) +} + func NewTSimpleJSONProtocolConf(t TTransport, conf *TConfiguration) *TSimpleJSONProtocol { PropagateTConfiguration(t, conf) v := &TSimpleJSONProtocol{ @@ -438,7 +446,8 @@ fmt.Errorf("size exceeded max allowed: %d", iSize), ) } - err = checkSizeForProtocol(int32(iSize), p.cfg) + minElemSize := p.getMinSerializedSize(keyType) + p.getMinSerializedSize(valueType) + err = checkContainerSizeForProtocol(iSize, minElemSize, p.RemainingBytes(), p.cfg) if err != nil { return keyType, valueType, 0, err } @@ -924,11 +933,37 @@ return nil } +// readBinaryBounded reads from p.reader up to and including delim, like +// bufio.Reader.ReadBytes(delim), but returns an error once the accumulated data +// exceeds the configured max message size. A JSON string or base64 value is +// delimiter-terminated and carries no declared length, so unlike a +// length-prefixed field it could otherwise be accumulated without bound. The +// size is re-checked after each buffered chunk, so an oversized value is +// rejected as it grows rather than buffered in full. +func (p *TSimpleJSONProtocol) readBinaryBounded(delim byte) ([]byte, error) { + var buf []byte + for { + chunk, err := p.reader.ReadSlice(delim) + buf = append(buf, chunk...) + if e := checkSizeForProtocol(int32(len(buf)), p.cfg); e != nil { + return nil, e + } + if err == nil { + return buf, nil + } + if err == bufio.ErrBufferFull { + continue + } + return buf, err + } +} + func (p *TSimpleJSONProtocol) ParseStringBody() (string, error) { - line, err := p.reader.ReadString(JSON_QUOTE) + buf, err := p.readBinaryBounded(JSON_QUOTE) if err != nil { return "", NewTProtocolException(err) } + line := string(buf) if endsWithoutEscapedQuote(line) { v, ok := jsonUnquote(string(JSON_QUOTE) + line) if !ok { @@ -953,11 +988,18 @@ var sb strings.Builder for { - line, err := p.reader.ReadString(JSON_QUOTE) + buf, err := p.readBinaryBounded(JSON_QUOTE) if err != nil { return "", NewTProtocolException(err) } + line := string(buf) sb.WriteString(line) + // readBinaryBounded caps each chunk; also cap the accumulated result so + // a string made of many short escaped-quote segments cannot sum past the + // limit. + if e := checkSizeForProtocol(int32(sb.Len()), p.cfg); e != nil { + return "", e + } if endsWithoutEscapedQuote(line) { return sb.String(), nil } @@ -976,7 +1018,7 @@ } func (p *TSimpleJSONProtocol) ParseBase64EncodedBody() ([]byte, error) { - line, err := p.reader.ReadBytes(JSON_QUOTE) + line, err := p.readBinaryBounded(JSON_QUOTE) if err != nil { return line, NewTProtocolException(err) } @@ -1066,10 +1108,11 @@ e := fmt.Errorf("Expected to be in the Object Context, but not in Object Context (%d)", cxt) return NewTProtocolExceptionWithType(INVALID_DATA, e) } - line, err := p.reader.ReadString(JSON_RBRACE[0]) + buf, err := p.readBinaryBounded(JSON_RBRACE[0]) if err != nil { return NewTProtocolException(err) } + line := string(buf) for _, char := range line { switch char { default: @@ -1104,6 +1147,41 @@ return isNull, NewTProtocolExceptionWithType(INVALID_DATA, err) } +func (p *TSimpleJSONProtocol) getMinSerializedSize(ttype TType) int32 { + switch ttype { + case STOP: + return 1 // T_STOP needs to count itself + case VOID: + return 1 // T_VOID needs to count itself + case BOOL: + return 1 // written as int + case BYTE: + return 1 + case DOUBLE: + return 1 + case I16: + return 1 + case I32: + return 1 + case I64: + return 1 + case STRING: + return 2 // empty string + case STRUCT: + return 2 // empty struct + case MAP: + return 2 // empty map + case SET: + return 2 // empty set + case LIST: + return 2 // empty list + case UUID: + return 16 // empty UUID + default: + return 1 // unknown type + } +} + func (p *TSimpleJSONProtocol) ParseElemListBegin() (elemType TType, size int, e error) { if isNull, e := p.ParseListBegin(); isNull || e != nil { return VOID, 0, e @@ -1123,7 +1201,8 @@ fmt.Errorf("size exceeded max allowed: %d", nSize), ) } - err = checkSizeForProtocol(int32(nSize), p.cfg) + minElemSize := p.getMinSerializedSize(elemType) + err = checkContainerSizeForProtocol(nSize, minElemSize, p.RemainingBytes(), p.cfg) if err != nil { return elemType, 0, err } @@ -1140,10 +1219,11 @@ e := fmt.Errorf("Expected to be in the List Context, but not in List Context (%d)", cxt) return NewTProtocolExceptionWithType(INVALID_DATA, e) } - line, err := p.reader.ReadString(JSON_RBRACKET[0]) + buf, err := p.readBinaryBounded(JSON_RBRACKET[0]) if err != nil { return NewTProtocolException(err) } + line := string(buf) for _, char := range line { switch char { default: @@ -1205,6 +1285,11 @@ continueFor := true inQuotes := false for continueFor { + // Numeric literals carry no declared length; bound the running byte + // count against the configured max message size as digits accumulate. + if e := checkSizeForProtocol(int32(buf.Len()), p.cfg); e != nil { + return NUMERIC_NULL, e + } c, err := p.reader.ReadByte() if err != nil { if errors.Is(err, io.EOF) { --- a/vendor/github.com/apache/thrift/lib/go/thrift/simple_server.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/simple_server.go @@ -245,7 +245,7 @@ } func (p *TSimpleServer) Serve() error { - p.logContext.CompareAndSwap(nil, Pointer(context.Background())) + p.logContext.CompareAndSwap(nil, new(context.Background())) err := p.Listen() if err != nil { --- a/vendor/github.com/apache/thrift/lib/go/thrift/socket.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/socket.go @@ -234,8 +234,7 @@ } func (p *TSocket) RemainingBytes() (num_bytes uint64) { - const maxSize = ^uint64(0) - return maxSize // the truth is, we just don't know unless framed is used + return UnknownRemainingBytes } var _ TConfigurationSetter = (*TSocket)(nil) --- a/vendor/github.com/apache/thrift/lib/go/thrift/ssl_server_socket.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/ssl_server_socket.go @@ -38,9 +38,8 @@ } func NewTSSLServerSocketTimeout(listenAddr string, cfg *tls.Config, clientTimeout time.Duration) (*TSSLServerSocket, error) { - if cfg.MinVersion == 0 { - cfg.MinVersion = tls.VersionTLS10 - } + // Leave an unset MinVersion alone so crypto/tls applies its own default + // floor rather than pinning an older minimum here. addr, err := net.ResolveTCPAddr("tcp", listenAddr) if err != nil { return nil, err --- a/vendor/github.com/apache/thrift/lib/go/thrift/ssl_socket.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/ssl_socket.go @@ -51,9 +51,8 @@ // } // }) func NewTSSLSocketConf(hostPort string, conf *TConfiguration) *TSSLSocket { - if cfg := conf.GetTLSConfig(); cfg != nil && cfg.MinVersion == 0 { - cfg.MinVersion = tls.VersionTLS10 - } + // Leave an unset MinVersion alone so crypto/tls applies its own default + // floor rather than pinning an older minimum here. return &TSSLSocket{ hostPort: hostPort, cfg: conf, @@ -255,8 +254,7 @@ } func (p *TSSLSocket) RemainingBytes() (num_bytes uint64) { - const maxSize = ^uint64(0) - return maxSize // the truth is, we just don't know unless framed is used + return UnknownRemainingBytes } var _ TConfigurationSetter = (*TSSLSocket)(nil) --- a/vendor/github.com/apache/thrift/lib/go/thrift/transport.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/transport.go @@ -35,10 +35,34 @@ Flush(ctx context.Context) (err error) } +// UnknownRemainingBytes is returned by RemainingBytes implementations that +// cannot tell how many bytes are still readable, for example an unframed +// socket. +const UnknownRemainingBytes = ^uint64(0) + type ReadSizeProvider interface { + // RemainingBytes returns the number of bytes that can still be read, + // or UnknownRemainingBytes when the transport has no way of knowing. + // + // The value is used as an upper bound when validating element counts + // read off the wire, so it must never be smaller than the number of + // bytes a well formed peer can still deliver for the current message, + // or valid messages will be rejected. A layered transport that cannot + // express its own readable bytes in terms of the transport below it + // (because it decompresses, or because it buffers ahead) must return + // UnknownRemainingBytes rather than pass the lower value through. RemainingBytes() (num_bytes uint64) } +// addRemainingBytes adds extra to a RemainingBytes value, saturating at +// UnknownRemainingBytes rather than wrapping around. +func addRemainingBytes(remaining, extra uint64) uint64 { + if remaining >= UnknownRemainingBytes-extra { + return UnknownRemainingBytes + } + return remaining + extra +} + // Encapsulates the I/O layer type TTransport interface { io.ReadWriteCloser --- a/vendor/github.com/apache/thrift/lib/go/thrift/transport_exception.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/transport_exception.go @@ -25,6 +25,8 @@ ) type timeoutable interface { + error + Timeout() bool } @@ -123,8 +125,7 @@ // // Note that this also includes TTransportException wrapped timeout errors. func isTimeoutError(err error) bool { - var t timeoutable - if errors.As(err, &t) { + if t, ok := errors.AsType[timeoutable](err); ok { return t.Timeout() } return false --- a/vendor/github.com/apache/thrift/lib/go/thrift/zlib_transport.go +++ b/vendor/github.com/apache/thrift/lib/go/thrift/zlib_transport.go @@ -30,6 +30,7 @@ type TZlibTransportFactory struct { level int factory TTransportFactory + cfg *TConfiguration } // TZlibTransport is a TTransport implementation that makes use of zlib compression. @@ -46,15 +47,24 @@ func (p *TZlibTransportFactory) GetTransport(trans TTransport) (TTransport, error) { if p.factory != nil { // wrap other factory + PropagateTConfiguration(trans, p.cfg) var err error trans, err = p.factory.GetTransport(trans) if err != nil { return nil, err } } - return NewTZlibTransport(trans, p.level) + return NewTZlibTransportConf(trans, p.level, p.cfg) } +// SetTConfiguration implements TConfigurationSetter for propagation. +func (p *TZlibTransportFactory) SetTConfiguration(conf *TConfiguration) { + PropagateTConfiguration(p.factory, conf) + p.cfg = conf +} + +var _ TConfigurationSetter = (*TZlibTransportFactory)(nil) + // NewTZlibTransportFactory constructs a new instance of NewTZlibTransportFactory func NewTZlibTransportFactory(level int) *TZlibTransportFactory { return &TZlibTransportFactory{level: level, factory: nil} @@ -66,6 +76,24 @@ return &TZlibTransportFactory{level: level, factory: factory} } +// NewTZlibTransportFactoryConf constructs a new instance of TZlibTransportFactory +// whose transports use conf, as a wrapper over factory if it is not nil +func NewTZlibTransportFactoryConf(level int, factory TTransportFactory, conf *TConfiguration) *TZlibTransportFactory { + PropagateTConfiguration(factory, conf) + return &TZlibTransportFactory{level: level, factory: factory, cfg: conf} +} + +// NewTZlibTransportConf constructs a new instance of TZlibTransport that uses +// conf, and passes conf on to trans +func NewTZlibTransportConf(trans TTransport, level int, conf *TConfiguration) (*TZlibTransport, error) { + z, err := NewTZlibTransport(trans, level) + if err != nil { + return nil, err + } + z.SetTConfiguration(conf) + return z, nil +} + // NewTZlibTransport constructs a new instance of TZlibTransport func NewTZlibTransport(trans TTransport, level int) (*TZlibTransport, error) { writer, closer, err := newZlibWriterCloserLevel(trans, level) @@ -136,8 +164,12 @@ // RemainingBytes returns the size in bytes of the data that is still to be // read. +// +// The underlying transport counts compressed bytes, which are no bound on the +// bytes this transport can still produce, so the answer is always +// UnknownRemainingBytes. func (z *TZlibTransport) RemainingBytes() uint64 { - return z.transport.RemainingBytes() + return UnknownRemainingBytes } func (z *TZlibTransport) Write(p []byte) (int, error) { --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -6,7 +6,9 @@ * [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy. * Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). * [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams. +* [lzw](https://github.com/klauspost/compress/tree/master/lzw) is a drop-in replacement for `compress/lzw` with 1.4-4x faster decompression and 1.1-2.7x faster compression, depending on the data. * [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. +* [Xpress](https://github.com/klauspost/compress/tree/master/xpress) decompression of the Microsoft XPRESS (MS-XCA) plain LZ77 and LZ77+Huffman formats (the LZ77+Huffman variant is the one used in WIM images and Windows Compact OS / WOF data). * [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently. * [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. @@ -27,6 +29,18 @@ # changelog +* Jul 1st, 2026 [1.19.0](https://github.com/klauspost/compress/releases/tag/v1.19.0) + * zstd: Add true concurrent stream encodingin https://github.com/klauspost/compress/pull/1136 + * zstd: arm64 decoder asm by @lizthegrey in https://github.com/klauspost/compress/pull/1160 + * flate: Add inflate checkpoints in https://github.com/klauspost/compress/pull/1154 + * zstd: avoid unused BuildDict encoder allocation by @snissn in https://github.com/klauspost/compress/pull/1147 + * snappy/s2: Limit length of varint in `decodedLen` by @eustas in https://github.com/klauspost/compress/pull/1148 + * gzhttp: match qvalue parameter case-insensitively (RFC 7231) by @z9z in https://github.com/klauspost/compress/pull/1149 + * zip: add NameDecoder callback for legacy encoding rewrite by @SAY-5 in https://github.com/klauspost/compress/pull/1150 + * huff0: Allow building tables from histogram in https://github.com/klauspost/compress/pull/1155 + * huff0: Allow building table from oversized histogram in https://github.com/klauspost/compress/pull/1156 + * s2sx: Clean symlink targets in https://github.com/klauspost/compress/pull/1163 + * Feb 9th, 2026 [1.18.4](https://github.com/klauspost/compress/releases/tag/v1.18.4) * gzhttp: Add zstandard to server handler wrapper https://github.com/klauspost/compress/pull/1121 * zstd: Add ResetWithOptions to encoder/decoder https://github.com/klauspost/compress/pull/1122 --- a/vendor/github.com/klauspost/compress/flate/deflate.go +++ b/vendor/github.com/klauspost/compress/flate/deflate.go @@ -95,6 +95,11 @@ index int hashMatch [maxMatchLength + minMatchLength]uint32 + // litBits[i] is the number of bits needed to encode window[index:i] + // as literals, where index is the window index at the start of the + // current deflateLazy call. Only allocated when d.chain > 100 (level 9). + litBits []int32 + // Input hash chains // hashHead[hashValue] contains the largest inputIndex with the specified hash value // If hashHead[hashValue] is within the current window, then @@ -260,6 +265,7 @@ } // Update window information. d.windowEnd += n + d.blockStart = d.windowEnd s.index = n } @@ -283,7 +289,9 @@ minIndex := max(pos-windowSize, 0) offset = 0 - if d.chain < 100 { + // The gain estimate below needs the tables that deflateLazy only + // generates when d.chain > 100, so the conditions must match. + if d.chain <= 100 { for i := prevHead; tries > 0; tries-- { if wEnd == win[i+length] { n := matchLen(win[i:i+minMatchLook], wPos) @@ -311,21 +319,25 @@ } // Minimum gain to accept a match. - cGain := 4 + cGain := int32(4) // Some like it higher (CSV), some like it lower (JSON) const baseCost = 3 // Base is 4 bytes at with an additional cost. // Matches must be better than this. + s := d.state + hashPrev := &s.hashPrev + hashOffset := s.hashOffset + litBits := s.litBits[pos : pos+minMatchLook+1] + for i := prevHead; tries > 0; tries-- { if wEnd == win[i+length] { n := matchLen(win[i:i+minMatchLook], wPos) if n > length { // Calculate gain. Estimate - newGain := d.h.bitLengthRaw(wPos[:n]) - int(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int(lengthExtraBits[lengthCodes[(n-3)&255]]) + newGain := litBits[n] - litBits[0] - int32(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int32(lengthExtraBits[lengthCodes[(n-3)&255]]) - //fmt.Println("gain:", newGain, "prev:", cGain, "raw:", d.h.bitLengthRaw(wPos[:n]), "this-len:", n, "prev-len:", length) if newGain > cGain { length = n offset = pos - i @@ -343,7 +355,7 @@ // hashPrev[i & windowMask] has already been overwritten, so stop now. break } - i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset + i = int(hashPrev[i&windowMask]) - hashOffset if i < minIndex { break } @@ -427,6 +439,20 @@ tmp[v]++ } d.h.generate(tmp[:], 15) + + // Compute the cumulative cost of encoding the window as literals, + // so findMatch can look up the cost of any range in constant time. + if s.litBits == nil { + s.litBits = make([]int32, 2*windowSize+1) + } + codes := d.h.codes[:256] + litBits := s.litBits[s.index : d.windowEnd+1] + total := int32(0) + litBits[0] = 0 + for i, v := range d.window[s.index:d.windowEnd] { + total += int32(codes[v].len()) + litBits[i+1] = total + } } s.maxInsertIndex = d.windowEnd - (minMatchLength - 1) --- a/vendor/github.com/klauspost/compress/flate/dict_decoder.go +++ b/vendor/github.com/klauspost/compress/flate/dict_decoder.go @@ -28,9 +28,10 @@ hist []byte // Sliding window history // Invariant: 0 <= rdPos <= wrPos <= len(hist) - wrPos int // Current output position in buffer - rdPos int // Have emitted hist[:rdPos] already - full bool // Has a full window length been written yet? + wrPos int // Current output position in buffer + rdPos int // Have emitted hist[:rdPos] already + flushed int64 // Total bytes returned by readFlush since init + full bool // Has a full window length been written yet? } // init initializes dictDecoder to have a sliding window dictionary of the given @@ -167,11 +168,22 @@ return dstPos - dstBase } +// appendWindow appends the current sliding window (up to len(hist) most recent +// bytes, oldest first) to dst. +func (dd *dictDecoder) appendWindow(dst []byte) []byte { + if dd.full { + dst = append(dst, dd.hist[dd.wrPos:]...) + return append(dst, dd.hist[:dd.wrPos]...) + } + return append(dst, dd.hist[:dd.wrPos]...) +} + // readFlush returns a slice of the historical buffer that is ready to be // emitted to the user. The data returned by readFlush must be fully consumed // before calling any other dictDecoder methods. func (dd *dictDecoder) readFlush() []byte { toRead := dd.hist[dd.rdPos:dd.wrPos] + dd.flushed += int64(len(toRead)) dd.rdPos = dd.wrPos if dd.wrPos == len(dd.hist) { dd.wrPos, dd.rdPos = 0, 0 @@ -179,3 +191,9 @@ } return toRead } + +// decoded reports the total number of bytes written into the dictionary since +// init (i.e. excluding any preset dict bytes). +func (dd *dictDecoder) decoded() int64 { + return dd.flushed + int64(dd.wrPos-dd.rdPos) +} --- a/vendor/github.com/klauspost/compress/flate/fast_encoder.go +++ b/vendor/github.com/klauspost/compress/flate/fast_encoder.go @@ -9,6 +9,7 @@ "fmt" "github.com/klauspost/compress/internal/le" + "github.com/klauspost/compress/internal/regmask" ) type fastEnc interface { @@ -108,7 +109,7 @@ // hash7 returns the hash of the lowest 7 bytes of u to fit in a hash table with h bits. // Preferably h should be a constant and should always be <64. func hash7(u uint64, h uint8) uint32 { - return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & reg8SizeMask64)) + return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & regmask.Shift64ByUint8)) } // hashLen returns a hash of the lowest mls bytes of with length output bits. --- a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go +++ b/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go @@ -32,14 +32,12 @@ // bufferFlushSize indicates the buffer size // after which bytes are flushed to the writer. - // Should preferably be a multiple of 6, since - // we accumulate 6 bytes between writes to the buffer. + // Between checks, at most 7 bytes are added to the buffer, + // and writes are done as unconditional 8-byte stores, so the + // buffer must have at least bufferFlushSize+7+8 bytes. bufferFlushSize = 246 ) -// Minimum length code that emits bits. -const lengthExtraBitsMinCode = 8 - // The number of extra bits needed by length code X - LENGTH_CODES_START. var lengthExtraBits = [32]uint8{ /* 257 */ 0, 0, 0, @@ -55,9 +53,6 @@ 64, 80, 96, 112, 128, 160, 192, 224, 255, } -// Minimum offset code that emits bits. -const offsetExtraBitsMinCode = 4 - // offset code word extra bits. var offsetExtraBits = [32]int8{ 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, @@ -67,8 +62,24 @@ 14, 14, } +// offsetCombined combines the number of extra bits and the base offset +// of each offset code in a single table: extra bits in the low 8 bits, +// and the base offset (already reduced by baseMatchOffset) in the upper bits. +// Entries for the extended window codes 30 and 31 are unused. var offsetCombined = [32]uint32{} +// lengthCombined combines, for each match length (reduced by +// baseMatchLength), the length code (relative to lengthCodesStart) in +// bits 0-4, the number of extra bits in bits 5-7 and the value of the +// extra bits in bits 8-12. +var lengthCombined = func() (t [256]uint32) { + for i := range t { + code := lengthCodes[i] + t[i] = uint32(code) | uint32(lengthExtraBits[code])<<5 | uint32(uint8(i)-lengthBase[code])<<8 + } + return t +}() + func init() { var offsetBase = [32]uint32{ /* normal deflate */ @@ -85,7 +96,7 @@ for i := range offsetCombined[:] { // Don't use extended window values... - if offsetExtraBits[i] == 0 || offsetBase[i] > 0x006000 { + if offsetBase[i] > 0x006000 { continue } offsetCombined[i] = uint32(offsetExtraBits[i]) | (offsetBase[i] << 8) @@ -115,10 +126,14 @@ lastHeader int // Set between 0 (reused block can be up to 2x the size) logNewTablePenalty uint - bytes [256 + 8]byte - literalFreq [lengthCodesStart + 32]uint16 - offsetFreq [32]uint16 - codegenFreq [codegenCodeCount]uint16 + + // bytes must hold at least bufferFlushSize+7+8 bytes (see bufferFlushSize). + // Its size is rounded up to a multiple of 8 to keep the following + // fields aligned, which is measurably faster in writeBlockHuff. + bytes [(bufferFlushSize + 7 + 8 + 7) &^ 7]byte + literalFreq [lengthCodesStart + 32]uint16 + offsetFreq [32]uint16 + codegenFreq [codegenCodeCount]uint16 // codegen must have an extra space for the final symbol. codegen [literalCount + offsetCodeCount + 1]uint8 @@ -842,124 +857,75 @@ lengths := leCodes[lengthCodesStart:] lengths = lengths[:32] - // Go 1.16 LOVES having these on stack. + // Keeping these on the stack instead of in w is significantly faster. bits, nbits, nbytes := w.bits, w.nbits, w.nbytes - for _, t := range tokens { - if t < 256 { - //w.writeCode(lits[t.literal()]) - c := lits[t] - bits |= c.code64() << (nbits & 63) - nbits += c.len() - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } - } - continue + // Flush whole bytes, so that nbits <= 7 when entering the loop below. + le.Store64(w.bytes[:], nbytes, bits) + nbytes += nbits >> 3 + bits >>= nbits & 56 + nbits &= 7 + if nbytes >= bufferFlushSize { + _, w.err = w.writer.Write(w.bytes[:nbytes]) + nbytes = 0 + if w.err != nil { + return } + } - // Write the length - length := t.length() - lengthCode := lengthCode(length) & 31 - if false { - w.writeCode(lengths[lengthCode]) - } else { - // inlined - c := lengths[lengthCode] + // The loop below flushes whole bytes at the end of every iteration, + // so that nbits <= 7 at the top of every iteration. + // A literal adds at most 15 bits and a match at most + // 15+5+15+13 = 48 bits, so up to three literals or a single match + // can be added to the 64-bit accumulator without an intermediate flush. + // Unconditionally storing 8 bytes and advancing by the number of + // whole bytes avoids a poorly predicted branch per token. + for i := 0; i < len(tokens); i++ { + t := tokens[i] + if t < 256 { + c := lits[t] bits |= c.code64() << (nbits & 63) nbits += c.len() - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 + // Add up to two more literals before flushing. + if i+1 < len(tokens) && tokens[i+1] < 256 { + i++ + c := lits[tokens[i]] + bits |= c.code64() << (nbits & 63) + nbits += c.len() + if i+1 < len(tokens) && tokens[i+1] < 256 { + i++ + c := lits[tokens[i]] + bits |= c.code64() << (nbits & 63) + nbits += c.len() } } - } - - if lengthCode >= lengthExtraBitsMinCode { - extraLengthBits := lengthExtraBits[lengthCode] - //w.writeBits(extraLength, extraLengthBits) - extraLength := int32(length - lengthBase[lengthCode]) - bits |= uint64(extraLength) << (nbits & 63) - nbits += extraLengthBits - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } - } - } - // Write the offset - offset := t.offset() - offsetCode := (offset >> 16) & 31 - if false { - w.writeCode(offs[offsetCode]) } else { - // inlined - c := offs[offsetCode] - bits |= c.code64() << (nbits & 63) - nbits += c.len() - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } - } - } - - if offsetCode >= offsetExtraBitsMinCode { - offsetComb := offsetCombined[offsetCode] - //w.writeBits(extraOffset, extraOffsetBits) - bits |= uint64((offset-(offsetComb>>8))&matchOffsetOnlyMask) << (nbits & 63) - nbits += uint8(offsetComb) - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } + // Write the length code and its extra bits as one unit. + lc := lengthCombined[t.length()] + c := lengths[lc&31] + bits |= (c.code64() | uint64(lc>>8)<<(c.len()&63)) << (nbits & 63) + nbits += c.len() + uint8(lc>>5)&7 + + // Write the offset code and its extra bits as one unit. + offset := t.offset() + offCode := (offset >> 16) & 31 + c = offs[offCode] + offComb := offsetCombined[offCode] + extra := (offset - (offComb >> 8)) & matchOffsetOnlyMask + bits |= (c.code64() | uint64(extra)<<(c.len()&63)) << (nbits & 63) + nbits += c.len() + uint8(offComb) + } + le.Store64(w.bytes[:], nbytes, bits) + nbytes += nbits >> 3 + bits >>= nbits & 56 + nbits &= 7 + if nbytes >= bufferFlushSize { + if w.err != nil { + nbytes = 0 + return } + _, w.err = w.writer.Write(w.bytes[:nbytes]) + nbytes = 0 } } // Restore... @@ -1094,8 +1060,8 @@ if debugDeflate { count -= int(nbytes)*8 + int(nbits) } - // Unroll, write 3 codes/loop. - // Fastest number of unrolls. + // Write 3 codes per iteration. The three codes are combined first, + // independently of the accumulator, and merged with a single shift. for len(input) > 3 { // We must have at least 48 bits free. if nbits >= 8 { @@ -1116,13 +1082,13 @@ _, w.err = w.writer.Write(w.bytes[:nbytes]) nbytes = 0 } - a, b := encoding[input[0]], encoding[input[1]] - bits |= a.code64() << (nbits & 63) - bits |= b.code64() << ((nbits + a.len()) & 63) - c := encoding[input[2]] - nbits += b.len() + a.len() - bits |= c.code64() << (nbits & 63) - nbits += c.len() + a, b, c := encoding[input[0]], encoding[input[1]], encoding[input[2]] + v := a.code64() | b.code64()<<(a.len()&63) + n := a.len() + b.len() + v |= c.code64() << (n & 63) + n += c.len() + bits |= v << (nbits & 63) + nbits += n input = input[3:] } --- a/vendor/github.com/klauspost/compress/flate/huffman_code.go +++ b/vendor/github.com/klauspost/compress/flate/huffman_code.go @@ -38,6 +38,9 @@ // Possible lengths are codegenCodeCount, offsetCodeCount and literalCount. // The largest of these is literalCount, so we allocate for that case. freqcache [literalCount + 1]literalNode + + // sortcache is scratch space for sortByFreq. + sortcache [literalCount + 1]literalNode } type literalNode struct { @@ -139,14 +142,6 @@ return total } -func (h *huffmanEncoder) bitLengthRaw(b []byte) int { - var total int - for _, f := range b { - total += int(h.codes[f].len()) - } - return total -} - // canReuseBits returns the number of bits or math.MaxInt32 if the encoder cannot be reused. func (h *huffmanEncoder) canReuseBits(freq []uint16) int { var total int @@ -309,27 +304,93 @@ } // Look at the leaves and assign them a bit count and an encoding as specified -// in RFC 1951 3.2.2 -func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) { - code := uint16(0) +// in RFC 1951 3.2.2. +// list must be sorted by increasing frequency and codes must have +// all entries for literals not in list set to zero. +func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode, codes []hcode) { + // The literals list[len(list)-bits] .. list[len(list)-1] + // are encoded using "bits" bits. Record the length of each literal + // temporarily in codes. for n, bits := range bitCount { - code <<= 1 if n == 0 || bits == 0 { continue } - // The literals list[len(list)-bits] .. list[len(list)-bits] - // are encoded using "bits" bits, and get the values - // code, code + 1, .... The code values are - // assigned in literal order (not frequency order). - chunk := list[len(list)-int(bits):] - - sortByLiteral(chunk) - for _, node := range chunk { - h.codes[node.literal] = newhcode(reverseBits(code, uint8(n)), uint8(n)) - code++ + for _, node := range list[len(list)-int(bits):] { + codes[node.literal] = hcode(n) } list = list[0 : len(list)-int(bits)] } + + // Code values are assigned in literal order (not frequency order) + // within each code length, starting from the first code of each length. + var nextCode [maxBitsLimit]uint16 + code := uint16(0) + for n := 1; n < len(bitCount); n++ { + code = (code + uint16(bitCount[n-1])) << 1 + nextCode[n] = code + } + for i, c := range codes { + if c == 0 { + continue + } + // n is at most maxBitsLimit-1; the mask only elides bounds checks. + n := uint8(c) + codes[i] = newhcode(reverseBits(nextCode[n&15], n), n) + nextCode[n&15]++ + } +} + +// sortByFreq sorts list by increasing frequency. +// The sort is stable, so nodes with equal frequencies keep their order. +func (h *huffmanEncoder) sortByFreq(list []literalNode) { + if len(list) <= 32 { + // Insertion sort is faster for small lists. + for i := 1; i < len(list); i++ { + n := list[i] + j := i + for ; j > 0 && list[j-1].freq > n.freq; j-- { + list[j] = list[j-1] + } + list[j] = n + } + return + } + + // Counting sort on the low byte and then on the high byte + // of the frequency. Both passes are stable. + tmp := h.sortcache[:len(list)] + var count [256]uint16 + var maxFreq uint16 + for _, n := range list { + count[n.freq&0xff]++ + maxFreq |= n.freq + } + var pos uint16 + for i, c := range count { + count[i] = pos + pos += c + } + for _, n := range list { + tmp[count[n.freq&0xff]] = n + count[n.freq&0xff]++ + } + if maxFreq < 256 { + copy(list, tmp) + return + } + count = [256]uint16{} + for _, n := range tmp { + count[n.freq>>8]++ + } + pos = 0 + for i, c := range count { + count[i] = pos + pos += c + } + for _, n := range tmp { + list[count[n.freq>>8]] = n + count[n.freq>>8]++ + } } // Update this Huffman Code object to be the minimum code for the specified frequency count. @@ -362,12 +423,14 @@ } return } - sortByFreq(list) + // list is in order of increasing literal value, so a stable sort + // by frequency orders it by (frequency, literal). + h.sortByFreq(list) // Get the number of literals for each bit count bitCount := h.bitCounts(list, maxBits) // And do the assignment - h.assignEncodingAndSize(bitCount, list) + h.assignEncodingAndSize(bitCount, list, codes) } // atLeastOne clamps the result between 1 and 15. @@ -394,24 +457,27 @@ } func histogramSplit(b []byte, h []uint16) { - // Tested, and slightly faster than 2-way. - // Writing to separate arrays and combining is also slightly slower. - h = h[:256] + // Walk four quarters in parallel, each into its own table. + // Tested to be faster than walking halves. Using separate tables + // avoids the load/store dependencies that arise when two of the + // streams hit the same byte value, which is common for text. + var h0, h1, h2, h3 [256]uint16 + // Make size divisible by 4 for len(b)&3 != 0 { - h[b[0]]++ + h0[b[0]]++ b = b[1:] } n := len(b) / 4 x, y, z, w := b[:n], b[n:], b[n+n:], b[n+n+n:] y, z, w = y[:len(x)], z[:len(x)], w[:len(x)] for i, t := range x { - v0 := &h[t] - v1 := &h[y[i]] - v2 := &h[z[i]] - v3 := &h[w[i]] - *v0++ - *v1++ - *v2++ - *v3++ + h0[t]++ + h1[y[i]]++ + h2[z[i]]++ + h3[w[i]]++ + } + h = h[:256] + for i := range h { + h[i] += h0[i] + h1[i] + h2[i] + h3[i] } } --- a/vendor/github.com/klauspost/compress/flate/inflate.go +++ b/vendor/github.com/klauspost/compress/flate/inflate.go @@ -14,6 +14,8 @@ "io" "math/bits" "sync" + + "github.com/klauspost/compress/internal/regmask" ) const ( @@ -342,6 +344,11 @@ final bool flushMode flushMode + cb func(InflateCheckpoint) + cp InflateCheckpoint + hasCP bool // WithResumeFrom was supplied + uncOffset int64 // baseline uncompressed offset (from a resume checkpoint) + cpBuf []byte } func (f *decompressor) nextBlock() { @@ -552,8 +559,8 @@ return err } } - rep += int(f.b & uint32(1<<(nb®SizeMaskUint32)-1)) - f.b >>= nb & regSizeMaskUint32 + rep += int(f.b & uint32(1<<(nb®mask.Shift32ByUint)-1)) + f.b >>= nb & regmask.Shift32ByUint f.nb -= nb if i+rep > n { if debugDecode { @@ -676,6 +683,18 @@ f.toRead = f.dict.readFlush() } + if f.cb != nil { + bitPos := f.roffset*8 - int64(f.nb) + f.cpBuf = f.dict.appendWindow(f.cpBuf[:0]) + f.cb(InflateCheckpoint{ + UncompressedOffset: f.uncOffset + f.dict.decoded(), + CompressedOffset: bitPos / 8, + Final: f.final, + BitOffset: uint8(bitPos & 7), + Window: f.cpBuf, + }) + } + f.step = nextBlock } @@ -714,7 +733,7 @@ return noEOF(err) } f.roffset++ - f.b |= uint32(c) << (f.nb & regSizeMaskUint32) + f.b |= uint32(c) << (f.nb & regmask.Shift32ByUint) f.nb += 8 return nil } @@ -739,7 +758,7 @@ return 0, noEOF(err) } f.roffset++ - b |= uint32(c) << (nb & regSizeMaskUint32) + b |= uint32(c) << (nb & regmask.Shift32ByUint) nb += 8 } chunk := h.chunks[b&(huffmanNumChunks-1)] @@ -758,7 +777,7 @@ f.err = CorruptInputError(f.roffset) return 0, f.err } - f.b = b >> (n & regSizeMaskUint32) + f.b = b >> (n & regmask.Shift32ByUint) f.nb = nb - n return int(chunk >> huffmanValueShift), nil } @@ -806,6 +825,45 @@ return nil } +// ResetCP will adjust the input to the provided checkpoint. +// It is assumed the input stream is forwarded to cp.CompressedOffset. +func (f *decompressor) ResetCP(r io.Reader, cp InflateCheckpoint) error { + *f = decompressor{ + r: makeReader(r), + bits: f.bits, + codebits: f.codebits, + h1: f.h1, + h2: f.h2, + dict: f.dict, + step: nextBlock, + cpBuf: f.cpBuf, + } + return f.applyCP(cp) +} + +// applyCP seeds the decompressor state from a resume checkpoint: +// loads the sliding window, sets the absolute compressed/uncompressed +// offsets, and skips cp.BitOffset bits into the first input byte so +// the next decode aligns with the start of a deflate block. +func (f *decompressor) applyCP(cp InflateCheckpoint) error { + f.dict.init(maxMatchOffset, cp.Window) + f.roffset = cp.CompressedOffset + f.uncOffset = cp.UncompressedOffset + f.final = cp.Final + f.b = 0 + f.nb = 0 + if cp.BitOffset > 0 { + c, err := f.r.ReadByte() + if err != nil { + return noEOF(err) + } + f.roffset++ + f.b = uint32(c) >> cp.BitOffset + f.nb = 8 - uint(cp.BitOffset) + } + return nil +} + type ReaderOpt func(*decompressor) // WithPartialBlock tells decompressor to return after each block, @@ -823,6 +881,36 @@ } } +// InflateCheckpoint provides a resumable checkpoint for inflate. +type InflateCheckpoint struct { + UncompressedOffset int64 // Byte offset in the decompressed stream + CompressedOffset int64 // Byte offset in the compressed stream + Final bool // True if this is the final block + BitOffset uint8 // 0-7 bits + Window []byte // 32KB sliding window dictionary +} + +// WithEobCallback will call the provided function after each block +// with the current gzip checkpoint. +// After returning the provided window can no longer be referenced. +// The callback will not be triggered after a block is marked "final". +// The callback is not retained after Reset. +func WithEobCallback(cb func(InflateCheckpoint)) ReaderOpt { + return func(f *decompressor) { + f.cb = cb + } +} + +// WithResumeFrom will adjust the input to the provided checkpoint. +// It is assumed the input stream is forwarded to the provided offset. +// The checkpoint is removed when Reset is called. +func WithResumeFrom(cp InflateCheckpoint) ReaderOpt { + return func(f *decompressor) { + f.cp = cp + f.hasCP = true + } +} + // NewReaderOpts returns new reader with provided options func NewReaderOpts(r io.Reader, opts ...ReaderOpt) io.ReadCloser { fixedHuffmanDecoderInit() @@ -838,6 +926,12 @@ opt(&f) } + if f.hasCP { + if err := f.applyCP(f.cp); err != nil { + f.err = err + } + } + return &f } --- a/vendor/github.com/klauspost/compress/flate/inflate_gen.go +++ b/vendor/github.com/klauspost/compress/flate/inflate_gen.go @@ -8,8 +8,19 @@ "fmt" "math/bits" "strings" + + "github.com/klauspost/compress/internal/regmask" ) +func peekBufio(fr *bufio.Reader) ([]byte, error) { + if fr.Buffered() == 0 { + if _, err := fr.Peek(1); err != nil && fr.Buffered() == 0 { + return nil, err + } + } + return fr.Peek(fr.Buffered()) +} + // Decode a single Huffman block from f. // hl and hd are the Huffman states for the lit/length values // and the distance values, respectively. If hd == nil, using the @@ -53,7 +64,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] @@ -71,7 +82,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n v = int(chunk >> huffmanValueShift) break @@ -109,15 +120,15 @@ if debugDecode { fmt.Println("morebits n>0:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 + fb >>= n & regmask.Shift32ByUint fnb -= n default: if debugDecode { @@ -137,11 +148,11 @@ if debugDecode { fmt.Println("morebits f.nb<5:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) @@ -165,7 +176,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] @@ -183,7 +194,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n dist = uint32(chunk >> huffmanValueShift) break @@ -197,7 +208,7 @@ case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) + extra := (dist & 1) << (nb & regmask.Shift32ByUint) for fnb < nb { c, err := fr.ReadByte() if err != nil { @@ -205,17 +216,17 @@ if debugDecode { fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fb >>= nb & regmask.Shift32ByUint fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + dist = 1<<((nb+1)®mask.Shift32ByUint) + 1 + extra // slower: dist = bitMask32[nb+1] + 2 + extra default: f.b, f.nb = fb, fnb @@ -304,7 +315,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] @@ -322,7 +333,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n v = int(chunk >> huffmanValueShift) break @@ -360,15 +371,15 @@ if debugDecode { fmt.Println("morebits n>0:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 + fb >>= n & regmask.Shift32ByUint fnb -= n default: if debugDecode { @@ -388,11 +399,11 @@ if debugDecode { fmt.Println("morebits f.nb<5:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) @@ -416,7 +427,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] @@ -434,7 +445,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n dist = uint32(chunk >> huffmanValueShift) break @@ -448,7 +459,7 @@ case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) + extra := (dist & 1) << (nb & regmask.Shift32ByUint) for fnb < nb { c, err := fr.ReadByte() if err != nil { @@ -456,17 +467,17 @@ if debugDecode { fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fb >>= nb & regmask.Shift32ByUint fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + dist = 1<<((nb+1)®mask.Shift32ByUint) + 1 + extra // slower: dist = bitMask32[nb+1] + 2 + extra default: f.b, f.nb = fb, fnb @@ -527,6 +538,8 @@ // but is smart enough to keep local variables in registers, so use nb and b, // inline call to moreBits and reassign b,nb back to f on return. fnb, fb, dict := f.nb, f.b, &f.dict + pbuf, _ := fr.Peek(fr.Buffered()) + pos := 0 switch f.stepState { case stateInit: @@ -548,14 +561,21 @@ n := uint(f.hl.maxRead) for { for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return + if pos >= len(pbuf) { + fr.Discard(pos) + var err error + pbuf, err = peekBufio(fr) + pos = 0 + if len(pbuf) == 0 { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } } + c := pbuf[pos] + pos++ f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] @@ -566,6 +586,7 @@ } if n <= fnb { if n == 0 { + fr.Discard(pos) f.b, f.nb = fb, fnb if debugDecode { fmt.Println("huffsym: n==0") @@ -573,7 +594,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n v = int(chunk >> huffmanValueShift) break @@ -586,6 +607,7 @@ case v < 256: dict.writeByte(byte(v)) if dict.availWrite() == 0 { + fr.Discard(pos) f.toRead = dict.readFlush() f.step = huffmanBufioReader f.stepState = stateInit @@ -594,6 +616,7 @@ } goto readLiteral case v == 256: + fr.Discard(pos) f.b, f.nb = fb, fnb f.finishBlock() return @@ -605,23 +628,31 @@ length = int(val.length) + 3 n := uint(val.extra) for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits n>0:", err) + if pos >= len(pbuf) { + fr.Discard(pos) + var err error + pbuf, err = peekBufio(fr) + pos = 0 + if len(pbuf) == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits n>0:", err) + } + f.err = noEOF(err) + return } - f.err = err - return } + c := pbuf[pos] + pos++ f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 + fb >>= n & regmask.Shift32ByUint fnb -= n default: + fr.Discard(pos) if debugDecode { fmt.Println(v, ">= maxNumLit") } @@ -633,17 +664,24 @@ var dist uint32 if f.hd == nil { for fnb < 5 { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb<5:", err) + if pos >= len(pbuf) { + fr.Discard(pos) + var err error + pbuf, err = peekBufio(fr) + pos = 0 + if len(pbuf) == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb<5:", err) + } + f.err = noEOF(err) + return } - f.err = err - return } + c := pbuf[pos] + pos++ f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) @@ -660,14 +698,21 @@ // inline call to moreBits and reassign b,nb back to f on return. for { for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return + if pos >= len(pbuf) { + fr.Discard(pos) + var err error + pbuf, err = peekBufio(fr) + pos = 0 + if len(pbuf) == 0 { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } } + c := pbuf[pos] + pos++ f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] @@ -678,6 +723,7 @@ } if n <= fnb { if n == 0 { + fr.Discard(pos) f.b, f.nb = fb, fnb if debugDecode { fmt.Println("huffsym: n==0") @@ -685,7 +731,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n dist = uint32(chunk >> huffmanValueShift) break @@ -699,27 +745,35 @@ case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) + extra := (dist & 1) << (nb & regmask.Shift32ByUint) for fnb < nb { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb= len(pbuf) { + fr.Discard(pos) + var err error + pbuf, err = peekBufio(fr) + pos = 0 + if len(pbuf) == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fb >>= nb & regmask.Shift32ByUint fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + dist = 1<<((nb+1)®mask.Shift32ByUint) + 1 + extra // slower: dist = bitMask32[nb+1] + 2 + extra default: + fr.Discard(pos) f.b, f.nb = fb, fnb if debugDecode { fmt.Println("dist too big:", dist, maxNumDist) @@ -730,6 +784,7 @@ // No check on length; encoding can be prescient. if dist > uint32(dict.histSize()) { + fr.Discard(pos) f.b, f.nb = fb, fnb if debugDecode { fmt.Println("dist > dict.histSize():", dist, dict.histSize()) @@ -752,6 +807,7 @@ f.copyLen -= cnt if dict.availWrite() == 0 || f.copyLen > 0 { + fr.Discard(pos) f.toRead = dict.readFlush() f.step = huffmanBufioReader // We need to continue this work f.stepState = stateDict @@ -806,7 +862,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] @@ -824,7 +880,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n v = int(chunk >> huffmanValueShift) break @@ -862,15 +918,15 @@ if debugDecode { fmt.Println("morebits n>0:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 + fb >>= n & regmask.Shift32ByUint fnb -= n default: if debugDecode { @@ -890,11 +946,11 @@ if debugDecode { fmt.Println("morebits f.nb<5:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) @@ -918,7 +974,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] @@ -936,7 +992,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n dist = uint32(chunk >> huffmanValueShift) break @@ -950,7 +1006,7 @@ case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) + extra := (dist & 1) << (nb & regmask.Shift32ByUint) for fnb < nb { c, err := fr.ReadByte() if err != nil { @@ -958,17 +1014,17 @@ if debugDecode { fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fb >>= nb & regmask.Shift32ByUint fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + dist = 1<<((nb+1)®mask.Shift32ByUint) + 1 + extra // slower: dist = bitMask32[nb+1] + 2 + extra default: f.b, f.nb = fb, fnb @@ -1057,7 +1113,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] @@ -1075,7 +1131,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n v = int(chunk >> huffmanValueShift) break @@ -1113,15 +1169,15 @@ if debugDecode { fmt.Println("morebits n>0:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 + fb >>= n & regmask.Shift32ByUint fnb -= n default: if debugDecode { @@ -1141,11 +1197,11 @@ if debugDecode { fmt.Println("morebits f.nb<5:", err) } - f.err = err + f.err = noEOF(err) return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) @@ -1169,7 +1225,7 @@ return } f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) + fb |= uint32(c) << (fnb & regmask.Shift32ByUint) fnb += 8 } chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] @@ -1187,7 +1243,7 @@ f.err = CorruptInputError(f.roffset) return } - fb = fb >> (n & regSizeMaskUint32) + fb = fb >> (n & regmask.Shift32ByUint) fnb = fnb - n dist = uint32(chunk >> huffmanValueShift) break @@ -1201,7 +1257,7 @@ case dist < maxNumDist: nb := uint(dist-2) >> 1 // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) + extra := (dist & 1) << (nb & regmask.Shift32ByUint) for fnb < nb { c, err := fr.ReadByte() if err != nil { @@ -1209,17 +1265,17 @@ if debugDecode { fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fb >>= nb & regmask.Shift32ByUint fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + dist = 1<<((nb+1)®mask.Shift32ByUint) + 1 + extra // slower: dist = bitMask32[nb+1] + 2 + extra default: f.b, f.nb = fb, fnb --- a/vendor/github.com/klauspost/compress/flate/matchlen_generic.go +++ b/vendor/github.com/klauspost/compress/flate/matchlen_generic.go @@ -12,18 +12,17 @@ // matchLen returns the maximum common prefix length of a and b. // a must be the shortest of the two. func matchLen(a, b []byte) (n int) { - left := len(a) - for left >= 8 { - diff := le.Load64(a, n) ^ le.Load64(b, n) + // Keeping a and b the same length and reslicing them as we go + // lets the compiler prove that no bounds checks are needed. + b = b[:len(a)] + for len(a) >= 8 { + diff := le.Load64(a, 0) ^ le.Load64(b, 0) if diff != 0 { return n + bits.TrailingZeros64(diff)>>3 } + a, b = a[8:], b[8:] n += 8 - left -= 8 } - - a = a[n:] - b = b[n:] for i := range a { if a[i] != b[i] { break --- a/vendor/github.com/klauspost/compress/flate/token.go +++ b/vendor/github.com/klauspost/compress/flate/token.go @@ -215,7 +215,11 @@ ux &= -0x7f800001 ux += 127 << 23 uval := math.Float32frombits(uint32(ux)) - log2 += ((-0.34484843)*uval+2.02466578)*uval - 0.67487759 + + p := float32(-0.34484843*uval) + 2.02466578 + q := float32(p*uval) - 0.67487759 + log2 += q + return log2 } @@ -232,7 +236,8 @@ for _, v := range t.litHist[:] { if v > 0 { n := float32(v) - shannon += atLeastOne(-mFastLog2(n*invTotal)) * n + w := float32(atLeastOne(-mFastLog2(n*invTotal)) * n) + shannon += w } } // Just add 15 for EOB @@ -240,7 +245,8 @@ for i, v := range t.extraHist[1 : literalCount-256] { if v > 0 { n := float32(v) - shannon += atLeastOne(-mFastLog2(n*invTotal)) * n + w := float32(atLeastOne(-mFastLog2(n*invTotal)) * n) + shannon += w bits += int(lengthExtraBits[i&31]) * int(v) nMatches += int(v) } @@ -251,7 +257,8 @@ for i, v := range t.offHist[:offsetCodeCount] { if v > 0 { n := float32(v) - shannon += atLeastOne(-mFastLog2(n*invTotal)) * n + w := float32(atLeastOne(-mFastLog2(n*invTotal)) * n) + shannon += w bits += int(offsetExtraBits[i&31]) * int(v) } } @@ -358,9 +365,6 @@ func (t token) length() uint8 { return uint8(t >> lengthShift) } -// Convert length to code. -func lengthCode(len uint8) uint8 { return lengthCodes[len] } - // Returns the offset code corresponding to a specific offset func offsetCode(off uint32) uint32 { if false { --- a/vendor/github.com/klauspost/compress/fse/compress.go +++ b/vendor/github.com/klauspost/compress/fse/compress.go @@ -659,7 +659,8 @@ if v >= 0 { total += int(v) } else { - total -= int(v) + // -1 takes one cell. Adding -v would let a wrapped -32768 pass. + total++ } } defer func() { --- a/vendor/github.com/klauspost/compress/fse/decompress.go +++ b/vendor/github.com/klauspost/compress/fse/decompress.go @@ -3,6 +3,7 @@ import ( "errors" "fmt" + "math" ) const ( @@ -123,6 +124,11 @@ remaining -= count gotTotal += count } + if count > math.MaxInt16 { + // Only one symbol filling a tableLog 15 table, which cannot be decoded. + // Truncated, it would leave the decoding table unwritten. + return fmt.Errorf("corruption detected (count %d > %d)", count, math.MaxInt16) + } s.norm[charnum&0xff] = int16(count) charnum++ previous0 = count == 0 --- a/vendor/github.com/klauspost/compress/huff0/bitreader.go +++ b/vendor/github.com/klauspost/compress/huff0/bitreader.go @@ -9,6 +9,7 @@ "errors" "fmt" "io" + "math/bits" "github.com/klauspost/compress/internal/le" ) @@ -210,6 +211,68 @@ return b.off*8 + uint(64-b.bitsRead) } +// canUseAsm reports whether the reader has a full 8-byte window ahead of +// its read pointer, which the Decompress4X asm loops need to take over. +func (b *bitReaderShifted) canUseAsm() bool { + return b.off >= 8 +} + +// prepareForAsm establishes the invariant the Decompress4X asm loops rely +// on: value == load64(in[off:off+8]) << bitsRead with bitsRead <= 7, so +// that a full group of symbols can never shift their sentinel bit out of +// the container. init leaves bitsRead == 8 when the final byte of the +// stream is exactly 0x01; that whole byte is consumed, so the same position +// is the window one byte lower with nothing consumed. Requires canUseAsm. +func (b *bitReaderShifted) prepareForAsm() { + if b.bitsRead >= 8 { + b.off-- + b.value = le.Load64(b.in, b.off) + b.bitsRead -= 8 + } +} + +// restoreFromAsm converts the state left behind by the Decompress4X asm +// loops back to the invariant the Go code relies on: value holds the 8 +// bytes at in[off:off+8] shifted left by bitsRead, with the low bitsRead +// bits zero. +// +// The asm keeps a sentinel bit in value just below the unread bits, so its +// trailing zero count is the number of consumed bits. The sentinel is ORed +// over the lowest bit of the window, which the loop never consumes before +// re-reading memory, so the window is re-read here too rather than taken +// from value. The asm reports off as the signed distance from the start of +// the stream, and it can be negative: a reload always reads a whole 8-byte +// window, so a stream that is nearly drained ends with up to 7 bytes of the +// previous stream (or the jump table) below its start inside the window. +// Those bytes sit below the stream's own bits and count as consumed. +// Anything further below, or more bits consumed than the stream holds, is +// corruption. +func (b *bitReaderShifted) restoreFromAsm() error { + off := int(b.off) + consumed := uint(bits.TrailingZeros64(b.value)) + if off < 0 { + if off < -7 { + return errors.New("corruption detected: stream underrun") + } + consumed += uint(-off) * 8 + if consumed > 64 { + return errors.New("corruption detected: stream underrun") + } + off = 0 + } + if off+8 > len(b.in) { + return errors.New("corruption detected: stream overrun") + } + b.off = uint(off) + b.bitsRead = uint8(consumed) + if consumed >= 64 { + b.value = 0 + } else { + b.value = le.Load64(b.in, b.off) << consumed + } + return nil +} + // close the bitstream and returns an error if out-of-buffer reads occurred. func (b *bitReaderShifted) close() error { // Release reference. --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -54,6 +54,10 @@ if len(in) < int(iSize) { return s, nil, fmt.Errorf("input too small for table, want %d bytes, have %d", iSize, len(in)) } + // The low nibble is the weights' FSE tableLog - 5, which the format limits to 6. + if in[0]&15+5 > 6 { + return s, nil, errors.New("corrupt input: weights tableLog too big") + } // FSE compressed weights s.fse.DecompressLimit = 255 hw := s.huffWeight[:] --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -1,222 +1,128 @@ //go:build amd64 && !appengine && !noasm && gc -// This file contains the specialisation of Decoder.Decompress4X -// and Decoder.Decompress1X that use an asm implementation of thir main loops. +// amd64 stubs and dispatch for the asm loops used by decompress_asm.go. package huff0 import ( - "errors" - "fmt" - "github.com/klauspost/compress/internal/cpuinfo" ) -// decompress4x_main_loop_x86 is an x86 assembler implementation -// of Decompress4X when tablelog > 8. +// decompress4x_main_loop_amd64 is an x86 assembler implementation +// of Decompress4X when tablelog > 8, decoding fastSymbols symbols per +// stream between bit container reloads. // //go:noescape func decompress4x_main_loop_amd64(ctx *decompress4xContext) -// decompress4x_8b_loop_x86 is an x86 assembler implementation -// of Decompress4X when tablelog <= 8 which decodes 4 entries -// per loop. +// decompress4x_8b_main_loop_amd64 is an x86 assembler implementation +// of Decompress4X when tablelog <= 8, decoding fast8bSymbols symbols +// per stream between bit container reloads. // //go:noescape func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) -// fallback8BitSize is the size where using Go version is faster. -const fallback8BitSize = 800 +// decompress4x_4b_main_loop_amd64 is an x86 assembler implementation +// of Decompress4X when tablelog <= 4, decoding fast4bSymbols symbols +// per stream between bit container reloads. +// +//go:noescape +func decompress4x_4b_main_loop_amd64(ctx *decompress4xContext) -type decompress4xContext struct { - pbr *[4]bitReaderShifted - peekBits uint8 - out *byte - dstEvery int - tbl *dEntrySingle - decoded int - limit *byte -} - -// Decompress4X will decompress a 4X encoded stream. -// The length of the supplied input must match the end of a block exactly. -// The *capacity* of the dst slice must match the destination size of -// the uncompressed data exactly. -func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - if len(src) < 6+(4*1) { - return nil, errors.New("input too small") - } - - use8BitTables := d.actualTableLog <= 8 - if cap(dst) < fallback8BitSize && use8BitTables { - return d.decompress4X8bit(dst, src) - } - - var br [4]bitReaderShifted - // Decode "jump table" - start := 6 - for i := range 3 { - length := int(src[i*2]) | (int(src[i*2+1]) << 8) - if start+length >= len(src) { - return nil, errors.New("truncated input (or invalid offset)") - } - err := br[i].init(src[start : start+length]) - if err != nil { - return nil, err - } - start += length - } - err := br[3].init(src[start:]) - if err != nil { - return nil, err - } - - // destination, offset to match first output - dstSize := cap(dst) - dst = dst[:dstSize] - out := dst - dstEvery := (dstSize + 3) / 4 - - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - single := d.dt.single[:tlSize] - - var decoded int - - if len(out) > 4*4 && !(br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4) { - ctx := decompress4xContext{ - pbr: &br, - peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() - out: &out[0], - dstEvery: dstEvery, - tbl: &single[0], - limit: &out[dstEvery-4], // Always stop decoding when first buffer gets here to avoid writing OOB on last. - } - if use8BitTables { - decompress4x_8b_main_loop_amd64(&ctx) - } else { - decompress4x_main_loop_amd64(&ctx) - } - - decoded = ctx.decoded - out = out[decoded/4:] - } - - // Decode remaining. - remainBytes := dstEvery - (decoded / 4) - for i := range br { - offset := dstEvery * i - endsAt := min(offset+remainBytes, len(out)) - br := &br[i] - bitsLeft := br.remaining() - for bitsLeft > 0 { - br.fill() - if offset >= endsAt { - return nil, errors.New("corruption detected: stream overrun 4") - } - - // Read value and increment offset. - val := br.peekBitsFast(d.actualTableLog) - v := single[val&tlMask].entry - nBits := uint8(v) - br.advance(nBits) - bitsLeft -= uint(nBits) - out[offset] = uint8(v >> 8) - offset++ - } - if offset != endsAt { - return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) - } - decoded += offset - dstEvery*i - err = br.close() - if err != nil { - return nil, err - } - } - if dstSize != decoded { - return nil, errors.New("corruption detected: short output block") - } - return dst, nil -} +// decompress4x_4b_main_loop_bmi2 is the BMI2 twin of decompress4x_4b_main_loop_amd64. +// +//go:noescape +func decompress4x_4b_main_loop_bmi2(ctx *decompress4xContext) + +// decompress4x_main_loop_bmi2 is the BMI2 twin of decompress4x_main_loop_amd64. +// +//go:noescape +func decompress4x_main_loop_bmi2(ctx *decompress4xContext) -// decompress4x_main_loop_x86 is an x86 assembler implementation -// of Decompress1X when tablelog > 8. +// decompress4x_8b_main_loop_bmi2 is the BMI2 twin of decompress4x_8b_main_loop_amd64. +// +//go:noescape +func decompress4x_8b_main_loop_bmi2(ctx *decompress4xContext) + +// decompress1x_main_loop_amd64 is an x86 assembler implementation +// of Decompress1X when tablelog > 8, decoding fastSymbols symbols +// between bit container reloads. // //go:noescape func decompress1x_main_loop_amd64(ctx *decompress1xContext) -// decompress4x_main_loop_x86 is an x86 with BMI2 assembler implementation -// of Decompress1X when tablelog > 8. +// decompress1x_8b_main_loop_amd64 is an x86 assembler implementation +// of Decompress1X when tablelog <= 8, decoding fast8bSymbols symbols +// between bit container reloads. +// +//go:noescape +func decompress1x_8b_main_loop_amd64(ctx *decompress1xContext) + +// decompress1x_4b_main_loop_amd64 is an x86 assembler implementation +// of Decompress1X when tablelog <= 4, decoding fast4bSymbols symbols +// between bit container reloads. +// +//go:noescape +func decompress1x_4b_main_loop_amd64(ctx *decompress1xContext) + +// decompress1x_main_loop_bmi2 is the BMI2 twin of decompress1x_main_loop_amd64. // //go:noescape func decompress1x_main_loop_bmi2(ctx *decompress1xContext) -type decompress1xContext struct { - pbr *bitReaderShifted - peekBits uint8 - out *byte - outCap int - tbl *dEntrySingle - decoded int -} - -// Error reported by asm implementations -const error_max_decoded_size_exeeded = -1 - -// Decompress1X will decompress a 1X encoded stream. -// The cap of the output buffer will be the maximum decompressed size. -// The length of the supplied input must match the end of a block exactly. -func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - var br bitReaderShifted - err := br.init(src) - if err != nil { - return dst, err - } - maxDecodedSize := cap(dst) - dst = dst[:maxDecodedSize] - - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - - if maxDecodedSize >= 4 { - ctx := decompress1xContext{ - pbr: &br, - out: &dst[0], - outCap: maxDecodedSize, - peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() - tbl: &d.dt.single[0], - } - - if cpuinfo.HasBMI2() { - decompress1x_main_loop_bmi2(&ctx) - } else { - decompress1x_main_loop_amd64(&ctx) - } - if ctx.decoded == error_max_decoded_size_exeeded { - return nil, ErrMaxDecodedSizeExceeded - } - - dst = dst[:ctx.decoded] - } - - // br < 8, so uint8 is fine - bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead - for bitsLeft > 0 { - br.fill() - if len(dst) >= maxDecodedSize { - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] - nBits := uint8(v.entry) - br.advance(nBits) - bitsLeft -= nBits - dst = append(dst, uint8(v.entry>>8)) +// decompress1x_8b_main_loop_bmi2 is the BMI2 twin of decompress1x_8b_main_loop_amd64. +// +//go:noescape +func decompress1x_8b_main_loop_bmi2(ctx *decompress1xContext) + +// decompress1x_4b_main_loop_bmi2 is the BMI2 twin of decompress1x_4b_main_loop_amd64. +// +//go:noescape +func decompress1x_4b_main_loop_bmi2(ctx *decompress1xContext) + +func decompress4x_main_loop_asm(ctx *decompress4xContext) { + if cpuinfo.HasBMI2() { + decompress4x_main_loop_bmi2(ctx) + } else { + decompress4x_main_loop_amd64(ctx) + } +} + +func decompress4x_8b_main_loop_asm(ctx *decompress4xContext) { + if cpuinfo.HasBMI2() { + decompress4x_8b_main_loop_bmi2(ctx) + } else { + decompress4x_8b_main_loop_amd64(ctx) + } +} + +func decompress4x_4b_main_loop_asm(ctx *decompress4xContext) { + if cpuinfo.HasBMI2() { + decompress4x_4b_main_loop_bmi2(ctx) + } else { + decompress4x_4b_main_loop_amd64(ctx) + } +} + +func decompress1x_main_loop_asm(ctx *decompress1xContext) { + if cpuinfo.HasBMI2() { + decompress1x_main_loop_bmi2(ctx) + } else { + decompress1x_main_loop_amd64(ctx) + } +} + +func decompress1x_8b_main_loop_asm(ctx *decompress1xContext) { + if cpuinfo.HasBMI2() { + decompress1x_8b_main_loop_bmi2(ctx) + } else { + decompress1x_8b_main_loop_amd64(ctx) + } +} + +func decompress1x_4b_main_loop_asm(ctx *decompress1xContext) { + if cpuinfo.HasBMI2() { + decompress1x_4b_main_loop_bmi2(ctx) + } else { + decompress1x_4b_main_loop_amd64(ctx) } - return dst, br.close() } --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s @@ -1,830 +1,3386 @@ -// Code generated by command: go run gen.go -out ../decompress_amd64.s -pkg=huff0. DO NOT EDIT. +// Code generated by command: go run gen.go -out ../decompress.s -arch amd64,arm64 -pkg=huff0. DO NOT EDIT. -//go:build amd64 && !appengine && !noasm && gc +//go:build !appengine && !noasm && gc // func decompress4x_main_loop_amd64(ctx *decompress4xContext) +// Requires: BMI, CMOV TEXT ·decompress4x_main_loop_amd64(SB), $0-8 - // Preload values - MOVQ ctx+0(FP), AX - MOVBQZX 8(AX), DI - MOVQ 16(AX), BX - MOVQ 48(AX), SI - MOVQ 24(AX), R8 - MOVQ 32(AX), R9 - MOVQ (AX), R10 - - // Main loop -main_loop: - XORL DX, DX - CMPQ BX, SI - SETGE DL - - // br0.fillFast32() - MOVQ 32(R10), R11 - MOVBQZX 40(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill0 - MOVQ 24(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ (R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 24(R10) - ORQ R13, R11 - - // exhausted += (br0.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill0: - // val0 := br0.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br0.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br0.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX - - // br0.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX) - - // update the bitreader structure - MOVQ R11, 32(R10) - MOVB R12, 40(R10) - - // br1.fillFast32() - MOVQ 80(R10), R11 - MOVBQZX 88(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill1 - MOVQ 72(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 48(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 72(R10) - ORQ R13, R11 - - // exhausted += (br1.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill1: - // val0 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br1.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br1.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX - - // br1.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX)(R8*1) - - // update the bitreader structure - MOVQ R11, 80(R10) - MOVB R12, 88(R10) - - // br2.fillFast32() - MOVQ 128(R10), R11 - MOVBQZX 136(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill2 - MOVQ 120(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 96(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 120(R10) - ORQ R13, R11 - - // exhausted += (br2.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill2: - // val0 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br2.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX - - // br2.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (BX)(R8*2) - - // update the bitreader structure - MOVQ R11, 128(R10) - MOVB R12, 136(R10) - - // br3.fillFast32() - MOVQ 176(R10), R11 - MOVBQZX 184(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill3 - MOVQ 168(R10), AX - SUBQ $0x20, R12 - SUBQ $0x04, AX - MOVQ 144(R10), R13 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R13*1), R13 - MOVQ R12, CX - SHLQ CL, R13 - MOVQ AX, 168(R10) - ORQ R13, R11 - - // exhausted += (br3.off < 4) - CMPQ AX, $0x04 - ADCB $+0, DL - -skip_fill3: - // val0 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 - - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX - - // br3.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - - // val1 := br3.peekTopBits(peekBits) - MOVQ DI, CX - MOVQ R11, R13 - SHRQ CL, R13 - - // v1 := table[val1&mask] - MOVW (R9)(R13*2), CX + MOVQ ctx+0(FP), AX - // br3.advance(uint8(v1.entry)) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - - // these two writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - LEAQ (R8)(R8*2), CX - MOVW AX, (BX)(CX*1) - - // update the bitreader structure - MOVQ R11, 176(R10) - MOVB R12, 184(R10) - ADDQ $0x02, BX - TESTB DL, DL - JZ main_loop - MOVQ ctx+0(FP), AX - SUBQ 16(AX), BX - SHLQ $0x02, BX - MOVQ BX, 40(AX) + // Preload values + MOVBQZX 8(AX), DX + MOVQ 32(AX), BX + MOVQ 16(AX), SI + MOVQ 24(AX), CX + LEAQ (SI)(CX*1), R8 + LEAQ (SI)(CX*2), R10 + LEAQ (CX)(CX*2), R12 + ADDQ SI, R12 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX) - - // update the bitreader structure - MOVQ R11, 32(R10) - MOVB R12, 40(R10) - - // br1.fillFast32() - MOVQ 80(R10), R11 - MOVBQZX 88(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill1 - MOVQ 72(R10), R13 - SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 48(R10), R14 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 - MOVQ R12, CX - SHLQ CL, R14 - MOVQ R13, 72(R10) - ORQ R14, R11 - - // exhausted += (br1.off < 4) - CMPQ R13, $0x04 - ADCB $+0, DL + TZCNTQ R13, R14 + MOVQ R14, CX + ANDQ $0x07, CX + SHRQ $0x03, R14 + MOVQ 88(AX), R13 + SUBQ R14, R13 + MOVQ R13, 88(AX) + MOVQ (R13), R13 + ORQ $0x01, R13 + SHLQ CL, R13 + ADDQ $0x0e, SI + ADDQ $0x0e, R8 + ADDQ $0x0e, R10 + ADDQ $0x0e, R12 + CMPQ SI, 96(AX) + JB inner_loop + JMP outer_loop + +done: + // Hand the state back: off = ip - in (negative when the window reached below the stream start), + // value = the sentinel-form container. bitReaderShifted.restoreFromAsm normalizes both. + MOVQ (AX), CX + MOVQ 64(AX), DX + SUBQ (CX), DX + MOVQ DX, 24(CX) + MOVQ DI, 32(CX) + MOVQ 72(AX), DX + SUBQ 48(CX), DX + MOVQ DX, 72(CX) + MOVQ R9, 80(CX) + MOVQ 80(AX), DX + SUBQ 96(CX), DX + MOVQ DX, 120(CX) + MOVQ R11, 128(CX) + MOVQ 88(AX), DX + SUBQ 144(CX), DX + MOVQ DX, 168(CX) + MOVQ R13, 176(CX) + SUBQ 16(AX), SI + SHLQ $0x02, SI + MOVQ SI, 40(AX) + RET -skip_fill1: - // val0 := br1.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 +// func decompress4x_main_loop_bmi2(ctx *decompress4xContext) +// Requires: BMI, BMI2, CMOV +TEXT ·decompress4x_main_loop_bmi2(SB), $0-8 + MOVQ ctx+0(FP), AX - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + LEAQ (BX)(SI*1), DI + LEAQ (BX)(SI*2), R9 + LEAQ (SI)(SI*2), R11 + ADDQ BX, R11 + + // Convert each bit reader to sentinel form: bits = value | 1<> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX)(R8*1) - - // update the bitreader structure - MOVQ R11, 80(R10) - MOVB R12, 88(R10) - - // br2.fillFast32() - MOVQ 128(R10), R11 - MOVBQZX 136(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill2 - MOVQ 120(R10), R13 - SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 96(R10), R14 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 - MOVQ R12, CX + // Refill the whole bytes consumed from below the window + MOVQ R11, R13 + ANDQ $0x38, R13 + MOVQ -8(R8), R14 + SHRQ $0x01, R14 + ANDQ $0x07, R11 + MOVQ R13, CX + XORQ $0x3f, CX + SHRQ CL, R14 + MOVQ R11, CX SHLQ CL, R14 - MOVQ R13, 120(R10) - ORQ R14, R11 + ORQ R14, R10 + SHRQ $0x03, R13 + SUBQ R13, R8 + ADDQ $0x05, SI + CMPQ SI, R12 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), R8 + MOVQ R8, 24(CX) + MOVQ R10, 32(CX) + MOVB R11, 40(CX) + SUBQ 16(AX), SI + MOVQ SI, 40(AX) + RET - // exhausted += (br2.off < 4) - CMPQ R13, $0x04 - ADCB $+0, DL +// func decompress1x_8b_main_loop_amd64(ctx *decompress1xContext) +// Requires: CMOV +TEXT ·decompress1x_8b_main_loop_amd64(SB), $0-8 + MOVQ ctx+0(FP), AX -skip_fill2: - // val0 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // Preload values + MOVBQZX 8(AX), DX + MOVQ 32(AX), BX + MOVQ 16(AX), SI + MOVQ 24(AX), DI + ADDQ SI, DI + MOVQ 56(AX), R8 + MOVQ 48(AX), R9 + MOVQ (AX), CX + MOVQ 32(CX), R10 + MOVBQZX 40(CX), R11 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 + MOVQ DI, CX + SUBQ SI, CX + JLE done + SHRQ $0x03, CX + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ R8, R12 + SUBQ R9, R12 + SHRQ $0x03, R12 + CMPQ R12, CX + CMOVQCS R12, CX + TESTQ CX, CX + JZ done + IMUL3Q $0x07, CX, R12 + ADDQ SI, R12 + +inner_loop: + // symbol 0 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, (SI) - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX + // symbol 1 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 1(SI) - // br2.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 + // symbol 2 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 2(SI) - // val1 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // symbol 3 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 3(SI) - // v1 := table[val0&mask] - MOVW (R9)(R13*2), CX + // symbol 4 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 4(SI) - // br2.advance(uint8(v1.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX + // symbol 5 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 5(SI) - // val2 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // symbol 6 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 6(SI) - // v2 := table[val0&mask] - MOVW (R9)(R13*2), CX + // Refill the whole bytes consumed from below the window + MOVQ R11, R13 + ANDQ $0x38, R13 + MOVQ -8(R8), R14 + SHRQ $0x01, R14 + ANDQ $0x07, R11 + MOVQ R13, CX + XORQ $0x3f, CX + SHRQ CL, R14 + MOVQ R11, CX + SHLQ CL, R14 + ORQ R14, R10 + SHRQ $0x03, R13 + SUBQ R13, R8 + ADDQ $0x07, SI + CMPQ SI, R12 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), R8 + MOVQ R8, 24(CX) + MOVQ R10, 32(CX) + MOVB R11, 40(CX) + SUBQ 16(AX), SI + MOVQ SI, 40(AX) + RET - // br2.advance(uint8(v2.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 +// func decompress1x_4b_main_loop_amd64(ctx *decompress1xContext) +// Requires: CMOV +TEXT ·decompress1x_4b_main_loop_amd64(SB), $0-8 + MOVQ ctx+0(FP), AX - // val3 := br2.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // Preload values + MOVBQZX 8(AX), DX + MOVQ 32(AX), BX + MOVQ 16(AX), SI + MOVQ 24(AX), DI + ADDQ SI, DI + MOVQ 56(AX), R8 + MOVQ 48(AX), R9 + MOVQ (AX), CX + MOVQ 32(CX), R10 + MOVBQZX 40(CX), R11 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 16 + MOVQ DI, CX + SUBQ SI, CX + JLE done + SHRQ $0x04, CX + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ R8, R12 + SUBQ R9, R12 + SHRQ $0x03, R12 + CMPQ R12, CX + CMOVQCS R12, CX + TESTQ CX, CX + JZ done + IMUL3Q $0x0e, CX, R12 + ADDQ SI, R12 + +inner_loop: + // symbol 0 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, (SI) - // v3 := table[val0&mask] - MOVW (R9)(R13*2), CX + // symbol 1 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 1(SI) - // br2.advance(uint8(v3.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX + // symbol 2 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 2(SI) - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (BX)(R8*2) - - // update the bitreader structure - MOVQ R11, 128(R10) - MOVB R12, 136(R10) - - // br3.fillFast32() - MOVQ 176(R10), R11 - MOVBQZX 184(R10), R12 - CMPQ R12, $0x20 - JBE skip_fill3 - MOVQ 168(R10), R13 - SUBQ $0x20, R12 - SUBQ $0x04, R13 - MOVQ 144(R10), R14 - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R13)(R14*1), R14 - MOVQ R12, CX - SHLQ CL, R14 - MOVQ R13, 168(R10) - ORQ R14, R11 + // symbol 3 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 3(SI) - // exhausted += (br3.off < 4) - CMPQ R13, $0x04 - ADCB $+0, DL + // symbol 4 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 4(SI) -skip_fill3: - // val0 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // symbol 5 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 5(SI) - // v0 := table[val0&mask] - MOVW (R9)(R13*2), CX + // symbol 6 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 6(SI) - // br3.advance(uint8(v0.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 + // symbol 7 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 7(SI) - // val1 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // symbol 8 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 8(SI) - // v1 := table[val0&mask] - MOVW (R9)(R13*2), CX + // symbol 9 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 9(SI) - // br3.advance(uint8(v1.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX + // symbol 10 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 10(SI) - // val2 := br3.peekTopBits(peekBits) - MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + // symbol 11 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 11(SI) - // v2 := table[val0&mask] - MOVW (R9)(R13*2), CX + // symbol 12 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 12(SI) - // br3.advance(uint8(v2.entry) - MOVB CH, AH - SHLQ CL, R11 - ADDB CL, R12 + // symbol 13 + MOVQ DX, CX + MOVQ R10, R13 + SHRQ CL, R13 + MOVWQZX (BX)(R13*2), CX + SHLQ CL, R10 + ADDQ CX, R11 + SHRQ $0x08, CX + MOVB CL, 13(SI) - // val3 := br3.peekTopBits(peekBits) + // Refill the whole bytes consumed from below the window MOVQ R11, R13 - MOVQ DI, CX - SHRQ CL, R13 + ANDQ $0x38, R13 + MOVQ -8(R8), R14 + SHRQ $0x01, R14 + ANDQ $0x07, R11 + MOVQ R13, CX + XORQ $0x3f, CX + SHRQ CL, R14 + MOVQ R11, CX + SHLQ CL, R14 + ORQ R14, R10 + SHRQ $0x03, R13 + SUBQ R13, R8 + ADDQ $0x0e, SI + CMPQ SI, R12 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), R8 + MOVQ R8, 24(CX) + MOVQ R10, 32(CX) + MOVB R11, 40(CX) + SUBQ 16(AX), SI + MOVQ SI, 40(AX) + RET - // v3 := table[val0&mask] - MOVW (R9)(R13*2), CX +// func decompress1x_main_loop_bmi2(ctx *decompress1xContext) +// Requires: BMI2, CMOV +TEXT ·decompress1x_main_loop_bmi2(SB), $0-8 + MOVQ ctx+0(FP), AX - // br3.advance(uint8(v3.entry) - MOVB CH, AL - SHLQ CL, R11 - ADDB CL, R12 - BSWAPL AX + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + ADDQ BX, SI + MOVQ 56(AX), DI + MOVQ 48(AX), R8 + MOVQ (AX), R10 + MOVQ 32(R10), R9 + MOVBQZX 40(R10), R10 - // these four writes get coalesced - // out[id * dstEvery + 0] = uint8(v0.entry >> 8) - // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - // out[id * dstEvery + 3] = uint8(v2.entry >> 8) - // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - LEAQ (R8)(R8*2), CX - MOVL AX, (BX)(CX*1) - - // update the bitreader structure - MOVQ R11, 176(R10) - MOVB R12, 184(R10) - ADDQ $0x04, BX - TESTB DL, DL - JZ main_loop - MOVQ ctx+0(FP), AX - SUBQ 16(AX), BX - SHLQ $0x02, BX - MOVQ BX, 40(AX) +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 + MOVQ SI, R11 + SUBQ BX, R11 + JLE done + SHRQ $0x03, R11 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ DI, R12 + SUBQ R8, R12 + SHRQ $0x03, R12 + CMPQ R12, R11 + CMOVQCS R12, R11 + TESTQ R11, R11 + JZ done + IMUL3Q $0x05, R11, R11 + ADDQ BX, R11 + +inner_loop: + // symbol 0 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, (BX) + + // symbol 1 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 1(BX) + + // symbol 2 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 2(BX) + + // symbol 3 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 3(BX) + + // symbol 4 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 4(BX) + + // Refill the whole bytes consumed from below the window + MOVQ R10, R12 + ANDQ $0x38, R12 + MOVQ -8(DI), R13 + SHRQ $0x01, R13 + ANDQ $0x07, R10 + MOVQ R12, R14 + XORQ $0x3f, R14 + SHRXQ R14, R13, R13 + SHLXQ R10, R13, R13 + ORQ R13, R9 + SHRQ $0x03, R12 + SUBQ R12, DI + ADDQ $0x05, BX + CMPQ BX, R11 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), DI + MOVQ DI, 24(CX) + MOVQ R9, 32(CX) + MOVB R10, 40(CX) + SUBQ 16(AX), BX + MOVQ BX, 40(AX) RET -// func decompress1x_main_loop_amd64(ctx *decompress1xContext) -TEXT ·decompress1x_main_loop_amd64(SB), $0-8 - MOVQ ctx+0(FP), CX - MOVQ 16(CX), DX - MOVQ 24(CX), BX - CMPQ BX, $0x04 - JB error_max_decoded_size_exceeded - LEAQ (DX)(BX*1), BX - MOVQ (CX), SI - MOVQ (SI), R8 - MOVQ 24(SI), R9 - MOVQ 32(SI), R10 - MOVBQZX 40(SI), R11 - MOVQ 32(CX), SI - MOVBQZX 8(CX), DI - JMP loop_condition - -main_loop: - // Check if we have room for 4 bytes in the output buffer - LEAQ 4(DX), CX - CMPQ CX, BX - JGE error_max_decoded_size_exceeded - - // Decode 4 values - CMPQ R11, $0x20 - JL bitReader_fillFast_1_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), R12 - MOVQ R11, CX - SHLQ CL, R12 - ORQ R12, R10 +// func decompress1x_8b_main_loop_bmi2(ctx *decompress1xContext) +// Requires: BMI2, CMOV +TEXT ·decompress1x_8b_main_loop_bmi2(SB), $0-8 + MOVQ ctx+0(FP), AX -bitReader_fillFast_1_end: - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AL - MOVBQZX CL, CX - ADDQ CX, R11 - SHLQ CL, R10 - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AH - MOVBQZX CL, CX - ADDQ CX, R11 - SHLQ CL, R10 - BSWAPL AX - CMPQ R11, $0x20 - JL bitReader_fillFast_2_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), R12 - MOVQ R11, CX - SHLQ CL, R12 - ORQ R12, R10 - -bitReader_fillFast_2_end: - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AH - MOVBQZX CL, CX - ADDQ CX, R11 - SHLQ CL, R10 - MOVQ DI, CX - MOVQ R10, R12 - SHRQ CL, R12 - MOVW (SI)(R12*2), CX - MOVB CH, AL - MOVBQZX CL, CX - ADDQ CX, R11 - SHLQ CL, R10 - BSWAPL AX - - // Store the decoded values - MOVL AX, (DX) - ADDQ $0x04, DX - -loop_condition: - CMPQ R9, $0x08 - JGE main_loop - - // Update ctx structure - MOVQ ctx+0(FP), AX - SUBQ 16(AX), DX - MOVQ DX, 40(AX) - MOVQ (AX), AX - MOVQ R9, 24(AX) - MOVQ R10, 32(AX) - MOVB R11, 40(AX) + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + ADDQ BX, SI + MOVQ 56(AX), DI + MOVQ 48(AX), R8 + MOVQ (AX), R10 + MOVQ 32(R10), R9 + MOVBQZX 40(R10), R10 + +outer_loop: + // Iterations allowed by the output: (limit - op) / 8 + MOVQ SI, R11 + SUBQ BX, R11 + JLE done + SHRQ $0x03, R11 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ DI, R12 + SUBQ R8, R12 + SHRQ $0x03, R12 + CMPQ R12, R11 + CMOVQCS R12, R11 + TESTQ R11, R11 + JZ done + IMUL3Q $0x07, R11, R11 + ADDQ BX, R11 + +inner_loop: + // symbol 0 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, (BX) + + // symbol 1 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 1(BX) + + // symbol 2 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 2(BX) + + // symbol 3 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 3(BX) + + // symbol 4 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 4(BX) + + // symbol 5 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 5(BX) + + // symbol 6 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 6(BX) + + // Refill the whole bytes consumed from below the window + MOVQ R10, R12 + ANDQ $0x38, R12 + MOVQ -8(DI), R13 + SHRQ $0x01, R13 + ANDQ $0x07, R10 + MOVQ R12, R14 + XORQ $0x3f, R14 + SHRXQ R14, R13, R13 + SHLXQ R10, R13, R13 + ORQ R13, R9 + SHRQ $0x03, R12 + SUBQ R12, DI + ADDQ $0x07, BX + CMPQ BX, R11 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), DI + MOVQ DI, 24(CX) + MOVQ R9, 32(CX) + MOVB R10, 40(CX) + SUBQ 16(AX), BX + MOVQ BX, 40(AX) RET - // Report error -error_max_decoded_size_exceeded: +// func decompress1x_4b_main_loop_bmi2(ctx *decompress1xContext) +// Requires: BMI2, CMOV +TEXT ·decompress1x_4b_main_loop_bmi2(SB), $0-8 MOVQ ctx+0(FP), AX - MOVQ $-1, CX - MOVQ CX, 40(AX) - RET -// func decompress1x_main_loop_bmi2(ctx *decompress1xContext) -// Requires: BMI2 -TEXT ·decompress1x_main_loop_bmi2(SB), $0-8 - MOVQ ctx+0(FP), CX - MOVQ 16(CX), DX - MOVQ 24(CX), BX - CMPQ BX, $0x04 - JB error_max_decoded_size_exceeded - LEAQ (DX)(BX*1), BX - MOVQ (CX), SI - MOVQ (SI), R8 - MOVQ 24(SI), R9 - MOVQ 32(SI), R10 - MOVBQZX 40(SI), R11 - MOVQ 32(CX), SI - MOVBQZX 8(CX), DI - JMP loop_condition - -main_loop: - // Check if we have room for 4 bytes in the output buffer - LEAQ 4(DX), CX - CMPQ CX, BX - JGE error_max_decoded_size_exceeded - - // Decode 4 values - CMPQ R11, $0x20 - JL bitReader_fillFast_1_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), CX - SHLXQ R11, CX, CX - ORQ CX, R10 - -bitReader_fillFast_1_end: - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AL - MOVBQZX CL, CX - ADDQ CX, R11 - SHLXQ CX, R10, R10 - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AH - MOVBQZX CL, CX - ADDQ CX, R11 - SHLXQ CX, R10, R10 - BSWAPL AX - CMPQ R11, $0x20 - JL bitReader_fillFast_2_end - SUBQ $0x20, R11 - SUBQ $0x04, R9 - MOVL (R8)(R9*1), CX - SHLXQ R11, CX, CX - ORQ CX, R10 - -bitReader_fillFast_2_end: - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AH - MOVBQZX CL, CX - ADDQ CX, R11 - SHLXQ CX, R10, R10 - SHRXQ DI, R10, CX - MOVW (SI)(CX*2), CX - MOVB CH, AL - MOVBQZX CL, CX - ADDQ CX, R11 - SHLXQ CX, R10, R10 - BSWAPL AX - - // Store the decoded values - MOVL AX, (DX) - ADDQ $0x04, DX - -loop_condition: - CMPQ R9, $0x08 - JGE main_loop - - // Update ctx structure - MOVQ ctx+0(FP), AX - SUBQ 16(AX), DX - MOVQ DX, 40(AX) - MOVQ (AX), AX - MOVQ R9, 24(AX) - MOVQ R10, 32(AX) - MOVB R11, 40(AX) - RET + // Preload values + MOVBQZX 8(AX), CX + MOVQ 32(AX), DX + MOVQ 16(AX), BX + MOVQ 24(AX), SI + ADDQ BX, SI + MOVQ 56(AX), DI + MOVQ 48(AX), R8 + MOVQ (AX), R10 + MOVQ 32(R10), R9 + MOVBQZX 40(R10), R10 - // Report error -error_max_decoded_size_exceeded: - MOVQ ctx+0(FP), AX - MOVQ $-1, CX - MOVQ CX, 40(AX) +outer_loop: + // Iterations allowed by the output: (limit - op) / 16 + MOVQ SI, R11 + SUBQ BX, R11 + JLE done + SHRQ $0x04, R11 + + // Iterations allowed by the input: a refill reads the 8 bytes below the window and + // moves it down by at most 7, so every read stays inside the stream while ip stays + // at least 8 above ilowest. + MOVQ DI, R12 + SUBQ R8, R12 + SHRQ $0x03, R12 + CMPQ R12, R11 + CMOVQCS R12, R11 + TESTQ R11, R11 + JZ done + IMUL3Q $0x0e, R11, R11 + ADDQ BX, R11 + +inner_loop: + // symbol 0 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, (BX) + + // symbol 1 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 1(BX) + + // symbol 2 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 2(BX) + + // symbol 3 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 3(BX) + + // symbol 4 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 4(BX) + + // symbol 5 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 5(BX) + + // symbol 6 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 6(BX) + + // symbol 7 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 7(BX) + + // symbol 8 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 8(BX) + + // symbol 9 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 9(BX) + + // symbol 10 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 10(BX) + + // symbol 11 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 11(BX) + + // symbol 12 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 12(BX) + + // symbol 13 + SHRXQ CX, R9, R12 + MOVWQZX (DX)(R12*2), R12 + SHLXQ R12, R9, R9 + ADDQ R12, R10 + SHRQ $0x08, R12 + MOVB R12, 13(BX) + + // Refill the whole bytes consumed from below the window + MOVQ R10, R12 + ANDQ $0x38, R12 + MOVQ -8(DI), R13 + SHRQ $0x01, R13 + ANDQ $0x07, R10 + MOVQ R12, R14 + XORQ $0x3f, R14 + SHRXQ R14, R13, R13 + SHLXQ R10, R13, R13 + ORQ R13, R9 + SHRQ $0x03, R12 + SUBQ R12, DI + ADDQ $0x0e, BX + CMPQ BX, R11 + JB inner_loop + JMP outer_loop + +done: + // Hand the state back in the Go bit reader's own form: off = ip - in, value, bitsRead. + MOVQ (AX), CX + SUBQ (CX), DI + MOVQ DI, 24(CX) + MOVQ R9, 32(CX) + MOVB R10, 40(CX) + SUBQ 16(AX), BX + MOVQ BX, 40(AX) RET --- a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || appengine || !gc || noasm // This file contains a generic implementation of Decoder.Decompress4X. package huff0 --- a/vendor/github.com/klauspost/compress/internal/snapref/decode.go +++ b/vendor/github.com/klauspost/compress/internal/snapref/decode.go @@ -31,7 +31,7 @@ // that the length header occupied. func decodedLen(src []byte) (blockLen, headerLen int, err error) { v, n := binary.Uvarint(src) - if n <= 0 || v > 0xffffffff { + if n <= 0 || n > 5 || v > 0xffffffff { return 0, 0, ErrCorrupt } --- a/vendor/github.com/klauspost/compress/s2/decode.go +++ b/vendor/github.com/klauspost/compress/s2/decode.go @@ -35,7 +35,7 @@ // that the length header occupied. func decodedLen(src []byte) (blockLen, headerLen int, err error) { v, n := binary.Uvarint(src) - if n <= 0 || v > 0xffffffff { + if n <= 0 || n > 5 || v > 0xffffffff { return 0, 0, ErrCorrupt } --- a/vendor/github.com/klauspost/compress/s2/encode_all.go +++ b/vendor/github.com/klauspost/compress/s2/encode_all.go @@ -981,7 +981,7 @@ cv = load64(src, s) continue } - } else if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) { + } else if repeat > 0 && uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) { base := s + checkRep // Extend back for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; { --- a/vendor/github.com/klauspost/compress/s2/encode_better.go +++ b/vendor/github.com/klauspost/compress/s2/encode_better.go @@ -86,12 +86,18 @@ // We initialize repeat to 0, so we never match on first attempt repeat := 0 + // Cap the adaptive skip, as encodeBetterBlockAsm and the snappy Go encoder + // below already do. Without a cap, a long stretch with no matches lets the + // skip grow geometrically until it strides past the start of the next + // compressible region and loses the matches there. + const maxSkip = 100 + for { candidateL := 0 nextS := 0 for { // Next src position to check - nextS = s + (s-nextEmit)>>7 + 1 + nextS = s + min((s-nextEmit)>>7+1, maxSkip) if nextS > sLimit { goto emitRemainder } --- a/vendor/github.com/klauspost/compress/s2/encode_go.go +++ b/vendor/github.com/klauspost/compress/s2/encode_go.go @@ -1,4 +1,4 @@ -//go:build !amd64 || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || appengine || !gc || noasm package s2 @@ -7,7 +7,7 @@ "math/bits" ) -const hasAmd64Asm = false +const hasAsm = false // encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It // assumes that the varint-encoded length of the decompressed bytes has already --- a/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s +++ b/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s @@ -1,6 +1,6 @@ -// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_amd64.go -pkg=s2. DO NOT EDIT. +// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_asm.go -pkg=s2. DO NOT EDIT. -//go:build !appengine && !noasm && gc && !noasm +//go:build (amd64 || arm64) && !appengine && !noasm && gc && !noasm #include "textflag.h" @@ -274,16 +274,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm matchlen_bsf_16repeat_extend_encodeBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm matchlen_match8_repeat_extend_encodeBlockAsm: CMPL R9, $0x08 @@ -296,16 +290,10 @@ JMP matchlen_match4_repeat_extend_encodeBlockAsm matchlen_bsf_8_repeat_extend_encodeBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm matchlen_match4_repeat_extend_encodeBlockAsm: CMPL R9, $0x04 @@ -898,16 +886,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBlockAsm matchlen_bsf_16match_nolit_encodeBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm matchlen_match8_match_nolit_encodeBlockAsm: CMPL DI, $0x08 @@ -920,16 +902,10 @@ JMP matchlen_match4_match_nolit_encodeBlockAsm matchlen_bsf_8_match_nolit_encodeBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm matchlen_match4_match_nolit_encodeBlockAsm: CMPL DI, $0x04 @@ -1689,16 +1665,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB matchlen_bsf_16repeat_extend_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm4MB + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm4MB matchlen_match8_repeat_extend_encodeBlockAsm4MB: CMPL R9, $0x08 @@ -1711,16 +1681,10 @@ JMP matchlen_match4_repeat_extend_encodeBlockAsm4MB matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm4MB + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm4MB matchlen_match4_repeat_extend_encodeBlockAsm4MB: CMPL R9, $0x04 @@ -2263,16 +2227,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBlockAsm4MB matchlen_bsf_16match_nolit_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm4MB + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm4MB matchlen_match8_match_nolit_encodeBlockAsm4MB: CMPL DI, $0x08 @@ -2285,16 +2243,10 @@ JMP matchlen_match4_match_nolit_encodeBlockAsm4MB matchlen_bsf_8_match_nolit_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm4MB + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm4MB matchlen_match4_match_nolit_encodeBlockAsm4MB: CMPL DI, $0x04 @@ -3006,16 +2958,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm12B matchlen_bsf_16repeat_extend_encodeBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm12B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm12B matchlen_match8_repeat_extend_encodeBlockAsm12B: CMPL R9, $0x08 @@ -3028,16 +2974,10 @@ JMP matchlen_match4_repeat_extend_encodeBlockAsm12B matchlen_bsf_8_repeat_extend_encodeBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm12B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm12B matchlen_match4_repeat_extend_encodeBlockAsm12B: CMPL R9, $0x04 @@ -3458,16 +3398,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBlockAsm12B matchlen_bsf_16match_nolit_encodeBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm12B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm12B matchlen_match8_match_nolit_encodeBlockAsm12B: CMPL DI, $0x08 @@ -3480,16 +3414,10 @@ JMP matchlen_match4_match_nolit_encodeBlockAsm12B matchlen_bsf_8_match_nolit_encodeBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm12B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm12B matchlen_match4_match_nolit_encodeBlockAsm12B: CMPL DI, $0x04 @@ -4091,16 +4019,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm10B matchlen_bsf_16repeat_extend_encodeBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm10B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm10B matchlen_match8_repeat_extend_encodeBlockAsm10B: CMPL R9, $0x08 @@ -4113,16 +4035,10 @@ JMP matchlen_match4_repeat_extend_encodeBlockAsm10B matchlen_bsf_8_repeat_extend_encodeBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm10B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm10B matchlen_match4_repeat_extend_encodeBlockAsm10B: CMPL R9, $0x04 @@ -4543,16 +4459,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBlockAsm10B matchlen_bsf_16match_nolit_encodeBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm10B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm10B matchlen_match8_match_nolit_encodeBlockAsm10B: CMPL DI, $0x08 @@ -4565,16 +4475,10 @@ JMP matchlen_match4_match_nolit_encodeBlockAsm10B matchlen_bsf_8_match_nolit_encodeBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm10B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm10B matchlen_match4_match_nolit_encodeBlockAsm10B: CMPL DI, $0x04 @@ -5176,16 +5080,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm8B matchlen_bsf_16repeat_extend_encodeBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm8B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm8B matchlen_match8_repeat_extend_encodeBlockAsm8B: CMPL R9, $0x08 @@ -5198,16 +5096,10 @@ JMP matchlen_match4_repeat_extend_encodeBlockAsm8B matchlen_bsf_8_repeat_extend_encodeBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm8B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm8B matchlen_match4_repeat_extend_encodeBlockAsm8B: CMPL R9, $0x04 @@ -5614,16 +5506,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBlockAsm8B matchlen_bsf_16match_nolit_encodeBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm8B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm8B matchlen_match8_match_nolit_encodeBlockAsm8B: CMPL DI, $0x08 @@ -5636,16 +5522,10 @@ JMP matchlen_match4_match_nolit_encodeBlockAsm8B matchlen_bsf_8_match_nolit_encodeBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm8B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm8B matchlen_match4_match_nolit_encodeBlockAsm8B: CMPL DI, $0x04 @@ -6140,16 +6020,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm matchlen_bsf_16match_nolit_encodeBetterBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm matchlen_match8_match_nolit_encodeBetterBlockAsm: CMPL R8, $0x08 @@ -6162,16 +6036,10 @@ JMP matchlen_match4_match_nolit_encodeBetterBlockAsm matchlen_bsf_8_match_nolit_encodeBetterBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm matchlen_match4_match_nolit_encodeBetterBlockAsm: CMPL R8, $0x04 @@ -7235,16 +7103,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm4MB + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm4MB matchlen_match8_match_nolit_encodeBetterBlockAsm4MB: CMPL R8, $0x08 @@ -7257,16 +7119,10 @@ JMP matchlen_match4_match_nolit_encodeBetterBlockAsm4MB matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm4MB + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm4MB matchlen_match4_match_nolit_encodeBetterBlockAsm4MB: CMPL R8, $0x04 @@ -8256,16 +8112,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm12B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm12B matchlen_match8_match_nolit_encodeBetterBlockAsm12B: CMPL R8, $0x08 @@ -8278,16 +8128,10 @@ JMP matchlen_match4_match_nolit_encodeBetterBlockAsm12B matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm12B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm12B matchlen_match4_match_nolit_encodeBetterBlockAsm12B: CMPL R8, $0x04 @@ -9130,16 +8974,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm10B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm10B matchlen_match8_match_nolit_encodeBetterBlockAsm10B: CMPL R8, $0x08 @@ -9152,16 +8990,10 @@ JMP matchlen_match4_match_nolit_encodeBetterBlockAsm10B matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm10B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm10B matchlen_match4_match_nolit_encodeBetterBlockAsm10B: CMPL R8, $0x04 @@ -10004,16 +9836,10 @@ JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm8B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm8B matchlen_match8_match_nolit_encodeBetterBlockAsm8B: CMPL R8, $0x08 @@ -10026,16 +9852,10 @@ JMP matchlen_match4_match_nolit_encodeBetterBlockAsm8B matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm8B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm8B matchlen_match4_match_nolit_encodeBetterBlockAsm8B: CMPL R8, $0x04 @@ -10985,16 +10805,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm matchlen_match8_repeat_extend_encodeSnappyBlockAsm: CMPL R8, $0x08 @@ -11007,16 +10821,10 @@ JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm matchlen_match4_repeat_extend_encodeSnappyBlockAsm: CMPL R8, $0x04 @@ -11329,16 +11137,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm matchlen_bsf_16match_nolit_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm matchlen_match8_match_nolit_encodeSnappyBlockAsm: CMPL DI, $0x08 @@ -11351,16 +11153,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm matchlen_match4_match_nolit_encodeSnappyBlockAsm: CMPL DI, $0x04 @@ -11901,16 +11697,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K: CMPL R8, $0x08 @@ -11923,16 +11713,10 @@ JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K: CMPL R8, $0x04 @@ -12205,16 +11989,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm64K + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm64K matchlen_match8_match_nolit_encodeSnappyBlockAsm64K: CMPL DI, $0x08 @@ -12227,16 +12005,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm64K matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm64K + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm64K matchlen_match4_match_nolit_encodeSnappyBlockAsm64K: CMPL DI, $0x04 @@ -12737,16 +12509,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B: CMPL R8, $0x08 @@ -12759,16 +12525,10 @@ JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B: CMPL R8, $0x04 @@ -13041,16 +12801,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm12B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm12B matchlen_match8_match_nolit_encodeSnappyBlockAsm12B: CMPL DI, $0x08 @@ -13063,16 +12817,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm12B matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm12B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm12B matchlen_match4_match_nolit_encodeSnappyBlockAsm12B: CMPL DI, $0x04 @@ -13573,16 +13321,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B: CMPL R8, $0x08 @@ -13595,16 +13337,10 @@ JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B: CMPL R8, $0x04 @@ -13877,16 +13613,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm10B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm10B matchlen_match8_match_nolit_encodeSnappyBlockAsm10B: CMPL DI, $0x08 @@ -13899,16 +13629,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm10B matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm10B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm10B matchlen_match4_match_nolit_encodeSnappyBlockAsm10B: CMPL DI, $0x04 @@ -14409,16 +14133,10 @@ JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B: CMPL R8, $0x08 @@ -14431,16 +14149,10 @@ JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B: CMPL R8, $0x04 @@ -14711,16 +14423,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm8B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm8B matchlen_match8_match_nolit_encodeSnappyBlockAsm8B: CMPL DI, $0x08 @@ -14733,16 +14439,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm8B matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm8B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm8B matchlen_match4_match_nolit_encodeSnappyBlockAsm8B: CMPL DI, $0x04 @@ -15144,16 +14844,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm: CMPL R8, $0x08 @@ -15166,16 +14860,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm: CMPL R8, $0x04 @@ -15790,16 +15478,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm64K + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm64K matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K: CMPL R8, $0x08 @@ -15812,16 +15494,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm64K + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm64K matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K: CMPL R8, $0x04 @@ -16371,16 +16047,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm12B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm12B matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B: CMPL R8, $0x08 @@ -16393,16 +16063,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm12B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm12B matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B: CMPL R8, $0x04 @@ -16952,16 +16616,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm10B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm10B matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B: CMPL R8, $0x08 @@ -16974,16 +16632,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm10B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm10B matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B: CMPL R8, $0x04 @@ -17533,16 +17185,10 @@ JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm8B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm8B matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B: CMPL R8, $0x08 @@ -17555,16 +17201,10 @@ JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm8B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm8B matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B: CMPL R8, $0x04 @@ -18138,16 +17778,10 @@ JMP matchlen_loopback_16_repeat_extend_calcBlockSize matchlen_bsf_16repeat_extend_calcBlockSize: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_calcBlockSize + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_calcBlockSize matchlen_match8_repeat_extend_calcBlockSize: CMPL R8, $0x08 @@ -18160,16 +17794,10 @@ JMP matchlen_match4_repeat_extend_calcBlockSize matchlen_bsf_8_repeat_extend_calcBlockSize: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_calcBlockSize + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_calcBlockSize matchlen_match4_repeat_extend_calcBlockSize: CMPL R8, $0x04 @@ -18371,16 +17999,10 @@ JMP matchlen_loopback_16_match_nolit_calcBlockSize matchlen_bsf_16match_nolit_calcBlockSize: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_calcBlockSize + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_calcBlockSize matchlen_match8_match_nolit_calcBlockSize: CMPL DI, $0x08 @@ -18393,16 +18015,10 @@ JMP matchlen_match4_match_nolit_calcBlockSize matchlen_bsf_8_match_nolit_calcBlockSize: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_calcBlockSize + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_calcBlockSize matchlen_match4_match_nolit_calcBlockSize: CMPL DI, $0x04 @@ -18723,16 +18339,10 @@ JMP matchlen_loopback_16_repeat_extend_calcBlockSizeSmall matchlen_bsf_16repeat_extend_calcBlockSizeSmall: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_calcBlockSizeSmall + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_calcBlockSizeSmall matchlen_match8_repeat_extend_calcBlockSizeSmall: CMPL R8, $0x08 @@ -18745,16 +18355,10 @@ JMP matchlen_match4_repeat_extend_calcBlockSizeSmall matchlen_bsf_8_repeat_extend_calcBlockSizeSmall: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_calcBlockSizeSmall + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_calcBlockSizeSmall matchlen_match4_repeat_extend_calcBlockSizeSmall: CMPL R8, $0x04 @@ -18926,16 +18530,10 @@ JMP matchlen_loopback_16_match_nolit_calcBlockSizeSmall matchlen_bsf_16match_nolit_calcBlockSizeSmall: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_calcBlockSizeSmall + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_calcBlockSizeSmall matchlen_match8_match_nolit_calcBlockSizeSmall: CMPL DI, $0x08 @@ -18948,16 +18546,10 @@ JMP matchlen_match4_match_nolit_calcBlockSizeSmall matchlen_bsf_8_match_nolit_calcBlockSizeSmall: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_calcBlockSizeSmall + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_calcBlockSizeSmall matchlen_match4_match_nolit_calcBlockSizeSmall: CMPL DI, $0x04 @@ -19737,16 +19329,10 @@ JMP matchlen_loopback_16_standalone matchlen_bsf_16standalone: -#ifdef GOAMD64_v3 TZCNTQ DI, DI - -#else - BSFQ DI, DI - -#endif - SARQ $0x03, DI - LEAL 8(SI)(DI*1), SI - JMP gen_match_len_end + SARQ $0x03, DI + LEAL 8(SI)(DI*1), SI + JMP gen_match_len_end matchlen_match8_standalone: CMPL DX, $0x08 @@ -19759,16 +19345,10 @@ JMP matchlen_match4_standalone matchlen_bsf_8_standalone: -#ifdef GOAMD64_v3 TZCNTQ BX, BX - -#else - BSFQ BX, BX - -#endif - SARQ $0x03, BX - LEAL (SI)(BX*1), SI - JMP gen_match_len_end + SARQ $0x03, BX + LEAL (SI)(BX*1), SI + JMP gen_match_len_end matchlen_match4_standalone: CMPL DX, $0x04 @@ -19802,7 +19382,7 @@ // func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) // Requires: SSE2 -TEXT ·cvtLZ4BlockAsm(SB), NOSPLIT, $0-64 +TEXT ·cvtLZ4BlockAsm(SB), NOSPLIT, $8-64 XORQ SI, SI MOVQ dst_base+0(FP), AX MOVQ dst_len+8(FP), CX @@ -19810,457 +19390,464 @@ MOVQ src_len+32(FP), BX LEAQ (DX)(BX*1), BX LEAQ -8(AX)(CX*1), CX - XORQ DI, DI + LEAQ -4(CX), DI + XORQ R8, R8 lz4_s2_loop: CMPQ DX, BX JAE lz4_s2_corrupt CMPQ AX, CX JAE lz4_s2_dstfull - MOVBQZX (DX), R8 - MOVQ R8, R9 - MOVQ R8, R10 - SHRQ $0x04, R9 - ANDQ $0x0f, R10 - CMPQ R8, $0xf0 + MOVBQZX (DX), R9 + MOVQ R9, R10 + MOVQ R9, R11 + SHRQ $0x04, R10 + ANDQ $0x0f, R11 + CMPQ R9, $0xf0 JB lz4_s2_ll_end lz4_s2_ll_loop: INCQ DX CMPQ DX, BX JAE lz4_s2_corrupt - MOVBQZX (DX), R8 - ADDQ R8, R9 - CMPQ R8, $0xff + MOVBQZX (DX), R9 + ADDQ R9, R10 + CMPQ R9, $0xff JEQ lz4_s2_ll_loop lz4_s2_ll_end: - LEAQ (DX)(R9*1), R8 - ADDQ $0x04, R10 - CMPQ R8, BX + LEAQ (DX)(R10*1), R9 + ADDQ $0x04, R11 + CMPQ R9, BX JAE lz4_s2_corrupt INCQ DX - INCQ R8 - TESTQ R9, R9 + INCQ R9 + TESTQ R10, R10 JZ lz4_s2_lits_done - LEAQ (AX)(R9*1), R11 - CMPQ R11, CX + LEAQ (AX)(R10*1), R12 + CMPQ R12, DI JAE lz4_s2_dstfull - ADDQ R9, SI - LEAL -1(R9), R11 - CMPL R11, $0x3c + ADDQ R10, SI + LEAL -1(R10), R12 + CMPL R12, $0x3c JB one_byte_lz4_s2 - CMPL R11, $0x00000100 + CMPL R12, $0x00000100 JB two_bytes_lz4_s2 - CMPL R11, $0x00010000 + CMPL R12, $0x00010000 JB three_bytes_lz4_s2 - CMPL R11, $0x01000000 + CMPL R12, $0x01000000 JB four_bytes_lz4_s2 MOVB $0xfc, (AX) - MOVL R11, 1(AX) + MOVL R12, 1(AX) ADDQ $0x05, AX JMP memmove_long_lz4_s2 four_bytes_lz4_s2: - MOVL R11, R12 - SHRL $0x10, R12 + MOVL R12, R13 + SHRL $0x10, R13 MOVB $0xf8, (AX) - MOVW R11, 1(AX) - MOVB R12, 3(AX) + MOVW R12, 1(AX) + MOVB R13, 3(AX) ADDQ $0x04, AX JMP memmove_long_lz4_s2 three_bytes_lz4_s2: MOVB $0xf4, (AX) - MOVW R11, 1(AX) + MOVW R12, 1(AX) ADDQ $0x03, AX JMP memmove_long_lz4_s2 two_bytes_lz4_s2: MOVB $0xf0, (AX) - MOVB R11, 1(AX) + MOVB R12, 1(AX) ADDQ $0x02, AX - CMPL R11, $0x40 + CMPL R12, $0x40 JB memmove_lz4_s2 JMP memmove_long_lz4_s2 one_byte_lz4_s2: - SHLB $0x02, R11 - MOVB R11, (AX) + SHLB $0x02, R12 + MOVB R12, (AX) ADDQ $0x01, AX memmove_lz4_s2: - LEAQ (AX)(R9*1), R11 + LEAQ (AX)(R10*1), R12 // genMemMoveShort - CMPQ R9, $0x08 + CMPQ R10, $0x08 JBE emit_lit_memmove_lz4_s2_memmove_move_8 - CMPQ R9, $0x10 + CMPQ R10, $0x10 JBE emit_lit_memmove_lz4_s2_memmove_move_8through16 - CMPQ R9, $0x20 + CMPQ R10, $0x20 JBE emit_lit_memmove_lz4_s2_memmove_move_17through32 JMP emit_lit_memmove_lz4_s2_memmove_move_33through64 emit_lit_memmove_lz4_s2_memmove_move_8: - MOVQ (DX), R12 - MOVQ R12, (AX) + MOVQ (DX), R13 + MOVQ R13, (AX) JMP memmove_end_copy_lz4_s2 emit_lit_memmove_lz4_s2_memmove_move_8through16: - MOVQ (DX), R12 - MOVQ -8(DX)(R9*1), DX - MOVQ R12, (AX) - MOVQ DX, -8(AX)(R9*1) + MOVQ (DX), R13 + MOVQ -8(DX)(R10*1), DX + MOVQ R13, (AX) + MOVQ DX, -8(AX)(R10*1) JMP memmove_end_copy_lz4_s2 emit_lit_memmove_lz4_s2_memmove_move_17through32: MOVOU (DX), X0 - MOVOU -16(DX)(R9*1), X1 + MOVOU -16(DX)(R10*1), X1 MOVOU X0, (AX) - MOVOU X1, -16(AX)(R9*1) + MOVOU X1, -16(AX)(R10*1) JMP memmove_end_copy_lz4_s2 emit_lit_memmove_lz4_s2_memmove_move_33through64: MOVOU (DX), X0 MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 + MOVOU -32(DX)(R10*1), X2 + MOVOU -16(DX)(R10*1), X3 MOVOU X0, (AX) MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) + MOVOU X2, -32(AX)(R10*1) + MOVOU X3, -16(AX)(R10*1) memmove_end_copy_lz4_s2: - MOVQ R11, AX + MOVQ R12, AX JMP lz4_s2_lits_emit_done memmove_long_lz4_s2: - LEAQ (AX)(R9*1), R11 + LEAQ (AX)(R10*1), R12 // genMemMoveLong MOVOU (DX), X0 MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ AX, R12 - ANDL $0x0000001f, R12 - MOVQ $0x00000040, R14 - SUBQ R12, R14 - DECQ R13 + MOVOU -32(DX)(R10*1), X2 + MOVOU -16(DX)(R10*1), X3 + MOVQ R10, R14 + SHRQ $0x05, R14 + MOVQ AX, R13 + ANDL $0x0000001f, R13 + MOVQ $0x00000040, R15 + SUBQ R13, R15 + DECQ R14 JA emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32 - LEAQ -32(DX)(R14*1), R12 - LEAQ -32(AX)(R14*1), R15 + LEAQ -32(DX)(R15*1), R13 + LEAQ -32(AX)(R15*1), BP emit_lit_memmove_long_lz4_s2large_big_loop_back: - MOVOU (R12), X4 - MOVOU 16(R12), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) + MOVOU (R13), X4 + MOVOU 16(R13), X5 + MOVOA X4, (BP) + MOVOA X5, 16(BP) + ADDQ $0x20, BP + ADDQ $0x20, R13 ADDQ $0x20, R15 - ADDQ $0x20, R12 - ADDQ $0x20, R14 - DECQ R13 + DECQ R14 JNA emit_lit_memmove_long_lz4_s2large_big_loop_back emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32: - MOVOU -32(DX)(R14*1), X4 - MOVOU -16(DX)(R14*1), X5 - MOVOA X4, -32(AX)(R14*1) - MOVOA X5, -16(AX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 + MOVOU -32(DX)(R15*1), X4 + MOVOU -16(DX)(R15*1), X5 + MOVOA X4, -32(AX)(R15*1) + MOVOA X5, -16(AX)(R15*1) + ADDQ $0x20, R15 + CMPQ R10, R15 JAE emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32 MOVOU X0, (AX) MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) - MOVQ R11, AX + MOVOU X2, -32(AX)(R10*1) + MOVOU X3, -16(AX)(R10*1) + MOVQ R12, AX lz4_s2_lits_emit_done: - MOVQ R8, DX + MOVQ R9, DX lz4_s2_lits_done: CMPQ DX, BX JNE lz4_s2_match - CMPQ R10, $0x04 + CMPQ R11, $0x04 JEQ lz4_s2_done JMP lz4_s2_corrupt lz4_s2_match: - LEAQ 2(DX), R8 - CMPQ R8, BX + LEAQ 2(DX), R9 + CMPQ R9, BX JAE lz4_s2_corrupt - MOVWQZX (DX), R9 - MOVQ R8, DX - TESTQ R9, R9 + MOVWQZX (DX), R10 + MOVQ R9, DX + TESTQ R10, R10 JZ lz4_s2_corrupt - CMPQ R9, SI + CMPQ R10, SI JA lz4_s2_corrupt - CMPQ R10, $0x13 + CMPQ R11, $0x13 JNE lz4_s2_ml_done lz4_s2_ml_loop: - MOVBQZX (DX), R8 + MOVBQZX (DX), R9 INCQ DX - ADDQ R8, R10 + ADDQ R9, R11 CMPQ DX, BX JAE lz4_s2_corrupt - CMPQ R8, $0xff + CMPQ R9, $0xff JEQ lz4_s2_ml_loop lz4_s2_ml_done: - ADDQ R10, SI - CMPQ R9, DI + ADDQ R11, SI + CMPQ R10, R8 JNE lz4_s2_docopy // emitRepeat emit_repeat_again_lz4_s2: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2 - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2 cant_repeat_two_offset_lz4_s2: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2 - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2 - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2 - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4_s2_loop JMP emit_repeat_again_lz4_s2 repeat_five_lz4_s2: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4_s2_loop repeat_four_lz4_s2: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4_s2_loop repeat_three_lz4_s2: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4_s2_loop repeat_two_lz4_s2: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop repeat_two_offset_lz4_s2: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop lz4_s2_docopy: - MOVQ R9, DI + MOVQ R10, R8 // emitCopy - CMPL R10, $0x40 + CMPL R11, $0x40 JBE two_byte_offset_short_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JAE long_offset_short_lz4_s2 - MOVL $0x00000001, R8 - LEAL 16(R8), R8 - MOVB R9, 1(AX) - MOVL R9, R11 - SHRL $0x08, R11 - SHLL $0x05, R11 - ORL R11, R8 - MOVB R8, (AX) + MOVL $0x00000001, R9 + LEAL 16(R9), R9 + MOVB R10, 1(AX) + MOVL R10, R12 + SHRL $0x08, R12 + SHLL $0x05, R12 + ORL R12, R9 + MOVB R9, (AX) ADDQ $0x02, AX - SUBL $0x08, R10 + SUBL $0x08, R11 // emitRepeat - LEAL -4(R10), R10 + LEAL -4(R11), R11 JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b emit_repeat_again_lz4_s2_emit_copy_short_2b: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2_emit_copy_short_2b - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2_emit_copy_short_2b - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2_emit_copy_short_2b cant_repeat_two_offset_lz4_s2_emit_copy_short_2b: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2_emit_copy_short_2b - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2_emit_copy_short_2b - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2_emit_copy_short_2b - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4_s2_loop JMP emit_repeat_again_lz4_s2_emit_copy_short_2b repeat_five_lz4_s2_emit_copy_short_2b: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4_s2_loop repeat_four_lz4_s2_emit_copy_short_2b: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4_s2_loop repeat_three_lz4_s2_emit_copy_short_2b: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4_s2_loop repeat_two_lz4_s2_emit_copy_short_2b: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop repeat_two_offset_lz4_s2_emit_copy_short_2b: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop long_offset_short_lz4_s2: MOVB $0xee, (AX) - MOVW R9, 1(AX) - LEAL -60(R10), R10 + MOVW R10, 1(AX) + LEAL -60(R11), R11 ADDQ $0x03, AX // emitRepeat emit_repeat_again_lz4_s2_emit_copy_short: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2_emit_copy_short - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2_emit_copy_short - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2_emit_copy_short cant_repeat_two_offset_lz4_s2_emit_copy_short: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2_emit_copy_short - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2_emit_copy_short - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2_emit_copy_short - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4_s2_loop JMP emit_repeat_again_lz4_s2_emit_copy_short repeat_five_lz4_s2_emit_copy_short: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4_s2_loop repeat_four_lz4_s2_emit_copy_short: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4_s2_loop repeat_three_lz4_s2_emit_copy_short: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4_s2_loop repeat_two_lz4_s2_emit_copy_short: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop repeat_two_offset_lz4_s2_emit_copy_short: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop two_byte_offset_short_lz4_s2: - MOVL R10, R8 - SHLL $0x02, R8 - CMPL R10, $0x0c + MOVL R11, R9 + SHLL $0x02, R9 + CMPL R11, $0x0c JAE emit_copy_three_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JAE emit_copy_three_lz4_s2 - LEAL -15(R8), R8 - MOVB R9, 1(AX) - SHRL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R8 - MOVB R8, (AX) + LEAL -15(R9), R9 + MOVB R10, 1(AX) + SHRL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R9 + MOVB R9, (AX) ADDQ $0x02, AX JMP lz4_s2_loop emit_copy_three_lz4_s2: - LEAL -2(R8), R8 - MOVB R8, (AX) - MOVW R9, 1(AX) + LEAL -2(R9), R9 + MOVB R9, (AX) + MOVW R10, 1(AX) ADDQ $0x03, AX JMP lz4_s2_loop @@ -20285,7 +19872,7 @@ // func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) // Requires: SSE2 -TEXT ·cvtLZ4sBlockAsm(SB), NOSPLIT, $0-64 +TEXT ·cvtLZ4sBlockAsm(SB), NOSPLIT, $8-64 XORQ SI, SI MOVQ dst_base+0(FP), AX MOVQ dst_len+8(FP), CX @@ -20293,459 +19880,466 @@ MOVQ src_len+32(FP), BX LEAQ (DX)(BX*1), BX LEAQ -8(AX)(CX*1), CX - XORQ DI, DI + LEAQ -4(CX), DI + XORQ R8, R8 lz4s_s2_loop: CMPQ DX, BX JAE lz4s_s2_corrupt CMPQ AX, CX JAE lz4s_s2_dstfull - MOVBQZX (DX), R8 - MOVQ R8, R9 - MOVQ R8, R10 - SHRQ $0x04, R9 - ANDQ $0x0f, R10 - CMPQ R8, $0xf0 + MOVBQZX (DX), R9 + MOVQ R9, R10 + MOVQ R9, R11 + SHRQ $0x04, R10 + ANDQ $0x0f, R11 + CMPQ R9, $0xf0 JB lz4s_s2_ll_end lz4s_s2_ll_loop: INCQ DX CMPQ DX, BX JAE lz4s_s2_corrupt - MOVBQZX (DX), R8 - ADDQ R8, R9 - CMPQ R8, $0xff + MOVBQZX (DX), R9 + ADDQ R9, R10 + CMPQ R9, $0xff JEQ lz4s_s2_ll_loop lz4s_s2_ll_end: - LEAQ (DX)(R9*1), R8 - ADDQ $0x03, R10 - CMPQ R8, BX + LEAQ (DX)(R10*1), R9 + ADDQ $0x03, R11 + CMPQ R9, BX JAE lz4s_s2_corrupt INCQ DX - INCQ R8 - TESTQ R9, R9 + INCQ R9 + TESTQ R10, R10 JZ lz4s_s2_lits_done - LEAQ (AX)(R9*1), R11 - CMPQ R11, CX + LEAQ (AX)(R10*1), R12 + CMPQ R12, DI JAE lz4s_s2_dstfull - ADDQ R9, SI - LEAL -1(R9), R11 - CMPL R11, $0x3c + ADDQ R10, SI + LEAL -1(R10), R12 + CMPL R12, $0x3c JB one_byte_lz4s_s2 - CMPL R11, $0x00000100 + CMPL R12, $0x00000100 JB two_bytes_lz4s_s2 - CMPL R11, $0x00010000 + CMPL R12, $0x00010000 JB three_bytes_lz4s_s2 - CMPL R11, $0x01000000 + CMPL R12, $0x01000000 JB four_bytes_lz4s_s2 MOVB $0xfc, (AX) - MOVL R11, 1(AX) + MOVL R12, 1(AX) ADDQ $0x05, AX JMP memmove_long_lz4s_s2 four_bytes_lz4s_s2: - MOVL R11, R12 - SHRL $0x10, R12 + MOVL R12, R13 + SHRL $0x10, R13 MOVB $0xf8, (AX) - MOVW R11, 1(AX) - MOVB R12, 3(AX) + MOVW R12, 1(AX) + MOVB R13, 3(AX) ADDQ $0x04, AX JMP memmove_long_lz4s_s2 three_bytes_lz4s_s2: MOVB $0xf4, (AX) - MOVW R11, 1(AX) + MOVW R12, 1(AX) ADDQ $0x03, AX JMP memmove_long_lz4s_s2 two_bytes_lz4s_s2: MOVB $0xf0, (AX) - MOVB R11, 1(AX) + MOVB R12, 1(AX) ADDQ $0x02, AX - CMPL R11, $0x40 + CMPL R12, $0x40 JB memmove_lz4s_s2 JMP memmove_long_lz4s_s2 one_byte_lz4s_s2: - SHLB $0x02, R11 - MOVB R11, (AX) + SHLB $0x02, R12 + MOVB R12, (AX) ADDQ $0x01, AX memmove_lz4s_s2: - LEAQ (AX)(R9*1), R11 + LEAQ (AX)(R10*1), R12 // genMemMoveShort - CMPQ R9, $0x08 + CMPQ R10, $0x08 JBE emit_lit_memmove_lz4s_s2_memmove_move_8 - CMPQ R9, $0x10 + CMPQ R10, $0x10 JBE emit_lit_memmove_lz4s_s2_memmove_move_8through16 - CMPQ R9, $0x20 + CMPQ R10, $0x20 JBE emit_lit_memmove_lz4s_s2_memmove_move_17through32 JMP emit_lit_memmove_lz4s_s2_memmove_move_33through64 emit_lit_memmove_lz4s_s2_memmove_move_8: - MOVQ (DX), R12 - MOVQ R12, (AX) + MOVQ (DX), R13 + MOVQ R13, (AX) JMP memmove_end_copy_lz4s_s2 emit_lit_memmove_lz4s_s2_memmove_move_8through16: - MOVQ (DX), R12 - MOVQ -8(DX)(R9*1), DX - MOVQ R12, (AX) - MOVQ DX, -8(AX)(R9*1) + MOVQ (DX), R13 + MOVQ -8(DX)(R10*1), DX + MOVQ R13, (AX) + MOVQ DX, -8(AX)(R10*1) JMP memmove_end_copy_lz4s_s2 emit_lit_memmove_lz4s_s2_memmove_move_17through32: MOVOU (DX), X0 - MOVOU -16(DX)(R9*1), X1 + MOVOU -16(DX)(R10*1), X1 MOVOU X0, (AX) - MOVOU X1, -16(AX)(R9*1) + MOVOU X1, -16(AX)(R10*1) JMP memmove_end_copy_lz4s_s2 emit_lit_memmove_lz4s_s2_memmove_move_33through64: MOVOU (DX), X0 MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 + MOVOU -32(DX)(R10*1), X2 + MOVOU -16(DX)(R10*1), X3 MOVOU X0, (AX) MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) + MOVOU X2, -32(AX)(R10*1) + MOVOU X3, -16(AX)(R10*1) memmove_end_copy_lz4s_s2: - MOVQ R11, AX + MOVQ R12, AX JMP lz4s_s2_lits_emit_done memmove_long_lz4s_s2: - LEAQ (AX)(R9*1), R11 + LEAQ (AX)(R10*1), R12 // genMemMoveLong MOVOU (DX), X0 MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ AX, R12 - ANDL $0x0000001f, R12 - MOVQ $0x00000040, R14 - SUBQ R12, R14 - DECQ R13 + MOVOU -32(DX)(R10*1), X2 + MOVOU -16(DX)(R10*1), X3 + MOVQ R10, R14 + SHRQ $0x05, R14 + MOVQ AX, R13 + ANDL $0x0000001f, R13 + MOVQ $0x00000040, R15 + SUBQ R13, R15 + DECQ R14 JA emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32 - LEAQ -32(DX)(R14*1), R12 - LEAQ -32(AX)(R14*1), R15 + LEAQ -32(DX)(R15*1), R13 + LEAQ -32(AX)(R15*1), BP emit_lit_memmove_long_lz4s_s2large_big_loop_back: - MOVOU (R12), X4 - MOVOU 16(R12), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) + MOVOU (R13), X4 + MOVOU 16(R13), X5 + MOVOA X4, (BP) + MOVOA X5, 16(BP) + ADDQ $0x20, BP + ADDQ $0x20, R13 ADDQ $0x20, R15 - ADDQ $0x20, R12 - ADDQ $0x20, R14 - DECQ R13 + DECQ R14 JNA emit_lit_memmove_long_lz4s_s2large_big_loop_back emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32: - MOVOU -32(DX)(R14*1), X4 - MOVOU -16(DX)(R14*1), X5 - MOVOA X4, -32(AX)(R14*1) - MOVOA X5, -16(AX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 + MOVOU -32(DX)(R15*1), X4 + MOVOU -16(DX)(R15*1), X5 + MOVOA X4, -32(AX)(R15*1) + MOVOA X5, -16(AX)(R15*1) + ADDQ $0x20, R15 + CMPQ R10, R15 JAE emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32 MOVOU X0, (AX) MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) - MOVQ R11, AX + MOVOU X2, -32(AX)(R10*1) + MOVOU X3, -16(AX)(R10*1) + MOVQ R12, AX lz4s_s2_lits_emit_done: - MOVQ R8, DX + MOVQ R9, DX lz4s_s2_lits_done: CMPQ DX, BX JNE lz4s_s2_match - CMPQ R10, $0x03 + CMPQ R11, $0x03 JEQ lz4s_s2_done JMP lz4s_s2_corrupt lz4s_s2_match: - CMPQ R10, $0x03 + CMPQ R11, $0x03 JEQ lz4s_s2_loop - LEAQ 2(DX), R8 - CMPQ R8, BX + LEAQ 2(DX), R9 + CMPQ R9, BX JAE lz4s_s2_corrupt - MOVWQZX (DX), R9 - MOVQ R8, DX - TESTQ R9, R9 + MOVWQZX (DX), R10 + MOVQ R9, DX + TESTQ R10, R10 JZ lz4s_s2_corrupt - CMPQ R9, SI + CMPQ R10, SI JA lz4s_s2_corrupt - CMPQ R10, $0x12 + CMPQ R11, $0x12 JNE lz4s_s2_ml_done lz4s_s2_ml_loop: - MOVBQZX (DX), R8 + MOVBQZX (DX), R9 INCQ DX - ADDQ R8, R10 + ADDQ R9, R11 CMPQ DX, BX JAE lz4s_s2_corrupt - CMPQ R8, $0xff + CMPQ R9, $0xff JEQ lz4s_s2_ml_loop lz4s_s2_ml_done: - ADDQ R10, SI - CMPQ R9, DI + ADDQ R11, SI + CMPQ R10, R8 JNE lz4s_s2_docopy // emitRepeat emit_repeat_again_lz4_s2: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2 - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2 cant_repeat_two_offset_lz4_s2: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2 - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2 - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2 - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4s_s2_loop JMP emit_repeat_again_lz4_s2 repeat_five_lz4_s2: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4s_s2_loop repeat_four_lz4_s2: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4s_s2_loop repeat_three_lz4_s2: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4s_s2_loop repeat_two_lz4_s2: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop repeat_two_offset_lz4_s2: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop lz4s_s2_docopy: - MOVQ R9, DI + MOVQ R10, R8 // emitCopy - CMPL R10, $0x40 + CMPL R11, $0x40 JBE two_byte_offset_short_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JAE long_offset_short_lz4_s2 - MOVL $0x00000001, R8 - LEAL 16(R8), R8 - MOVB R9, 1(AX) - MOVL R9, R11 - SHRL $0x08, R11 - SHLL $0x05, R11 - ORL R11, R8 - MOVB R8, (AX) + MOVL $0x00000001, R9 + LEAL 16(R9), R9 + MOVB R10, 1(AX) + MOVL R10, R12 + SHRL $0x08, R12 + SHLL $0x05, R12 + ORL R12, R9 + MOVB R9, (AX) ADDQ $0x02, AX - SUBL $0x08, R10 + SUBL $0x08, R11 // emitRepeat - LEAL -4(R10), R10 + LEAL -4(R11), R11 JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b emit_repeat_again_lz4_s2_emit_copy_short_2b: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2_emit_copy_short_2b - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2_emit_copy_short_2b - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2_emit_copy_short_2b cant_repeat_two_offset_lz4_s2_emit_copy_short_2b: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2_emit_copy_short_2b - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2_emit_copy_short_2b - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2_emit_copy_short_2b - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4s_s2_loop JMP emit_repeat_again_lz4_s2_emit_copy_short_2b repeat_five_lz4_s2_emit_copy_short_2b: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4s_s2_loop repeat_four_lz4_s2_emit_copy_short_2b: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4s_s2_loop repeat_three_lz4_s2_emit_copy_short_2b: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4s_s2_loop repeat_two_lz4_s2_emit_copy_short_2b: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop repeat_two_offset_lz4_s2_emit_copy_short_2b: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop long_offset_short_lz4_s2: MOVB $0xee, (AX) - MOVW R9, 1(AX) - LEAL -60(R10), R10 + MOVW R10, 1(AX) + LEAL -60(R11), R11 ADDQ $0x03, AX // emitRepeat emit_repeat_again_lz4_s2_emit_copy_short: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2_emit_copy_short - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2_emit_copy_short - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2_emit_copy_short cant_repeat_two_offset_lz4_s2_emit_copy_short: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2_emit_copy_short - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2_emit_copy_short - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2_emit_copy_short - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4s_s2_loop JMP emit_repeat_again_lz4_s2_emit_copy_short repeat_five_lz4_s2_emit_copy_short: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4s_s2_loop repeat_four_lz4_s2_emit_copy_short: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4s_s2_loop repeat_three_lz4_s2_emit_copy_short: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4s_s2_loop repeat_two_lz4_s2_emit_copy_short: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop repeat_two_offset_lz4_s2_emit_copy_short: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop two_byte_offset_short_lz4_s2: - MOVL R10, R8 - SHLL $0x02, R8 - CMPL R10, $0x0c + MOVL R11, R9 + SHLL $0x02, R9 + CMPL R11, $0x0c JAE emit_copy_three_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JAE emit_copy_three_lz4_s2 - LEAL -15(R8), R8 - MOVB R9, 1(AX) - SHRL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R8 - MOVB R8, (AX) + LEAL -15(R9), R9 + MOVB R10, 1(AX) + SHRL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R9 + MOVB R9, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop emit_copy_three_lz4_s2: - LEAL -2(R8), R8 - MOVB R8, (AX) - MOVW R9, 1(AX) + LEAL -2(R9), R9 + MOVB R9, (AX) + MOVW R10, 1(AX) ADDQ $0x03, AX JMP lz4s_s2_loop --- a/vendor/github.com/klauspost/compress/s2/hashtable_pool.go +++ b/vendor/github.com/klauspost/compress/s2/hashtable_pool.go @@ -25,7 +25,7 @@ sTable [betterShortTableSize]uint32 } -var betterTablePool = sync.Pool{New: func() interface{} { return &betterTables{} }} +var betterTablePool = sync.Pool{New: func() any { return &betterTables{} }} // betterSnappyTables holds better-snappy compression hash tables. type betterSnappyTables struct { @@ -33,7 +33,7 @@ sTable [betterShortTableSize]uint32 } -var betterSnappyTablePool = sync.Pool{New: func() interface{} { return &betterSnappyTables{} }} +var betterSnappyTablePool = sync.Pool{New: func() any { return &betterSnappyTables{} }} // bestTables holds best compression hash tables. type bestTables struct { @@ -41,7 +41,7 @@ sTable [bestShortTableSize]uint64 } -var bestTablePool = sync.Pool{New: func() interface{} { return &bestTables{} }} +var bestTablePool = sync.Pool{New: func() any { return &bestTables{} }} // getBetterTables gets a zeroed betterTables from the pool. func getBetterTables() *betterTables { --- a/vendor/github.com/klauspost/compress/s2/index.go +++ b/vendor/github.com/klauspost/compress/s2/index.go @@ -151,7 +151,7 @@ i.estBlockUncomp += i.estBlockUncomp * int64(removeN) } -func (i *Index) appendTo(b []byte, uncompTotal, compTotal int64) []byte { +func (i *Index) appendTo(b []byte, compTotal, uncompTotal int64) []byte { i.reduce() var tmp [binary.MaxVarintLen64]byte @@ -425,7 +425,7 @@ _, err := io.ReadFull(r, buf[:4]) if err != nil { if err == io.EOF { - return i.appendTo(nil, i.TotalUncompressed, i.TotalCompressed), nil + return i.appendTo(nil, i.TotalCompressed, i.TotalUncompressed), nil } return nil, err } --- a/vendor/github.com/klauspost/compress/s2/lz4convert.go +++ b/vendor/github.com/klauspost/compress/s2/lz4convert.go @@ -13,6 +13,7 @@ // LZ4Converter provides conversion from LZ4 blocks as defined here: // https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md type LZ4Converter struct { + noAsm bool } // ErrDstTooSmall is returned when provided destination is too small. @@ -32,7 +33,7 @@ s, d := 0, len(dst) dst = dst[:cap(dst)] - if !debug && hasAmd64Asm { + if !debug && hasAsm && !l.noAsm { res, sz := cvtLZ4BlockAsm(dst[d:], src) if res < 0 { const ( @@ -96,7 +97,8 @@ } s++ if ll > 0 { - if d+ll > dLimit { + // Leave room for a 5-byte literal header and an 8-byte copy. + if d+ll > len(dst)-13 { return nil, 0, ErrDstTooSmall } if debug { @@ -209,11 +211,13 @@ dst[2] = uint8(length >> 0) dst[1] = 0 dst[0] = 7<<2 | tagCopy1 + d += 5 if left > 0 { - d += 5 + emitRepeat16(dst[5:], offset, left) - break + // The loop condition checks room for the next chunk. + dst = dst[5:] + length = left + continue } - d += 5 break } } @@ -288,7 +292,7 @@ s, d := 0, len(dst) dst = dst[:cap(dst)] // Use assembly when possible - if !debug && hasAmd64Asm { + if !debug && hasAsm && !l.noAsm { res, sz := cvtLZ4BlockSnappyAsm(dst[d:], src) if res < 0 { const ( @@ -488,6 +492,11 @@ length -= 1 << 16 left := 0 if length > maxRepeat { + if len(dst) < 10 { + // No room for this and the next chunk. + // Report more than len(dst) so the caller sees dst is full. + return len(dst) + 1 + } left = length - maxRepeat + 4 length = maxRepeat - 4 } --- a/vendor/github.com/klauspost/compress/s2/lz4sconvert.go +++ b/vendor/github.com/klauspost/compress/s2/lz4sconvert.go @@ -21,6 +21,7 @@ // For Min Match of 4 bytes, Copy length value 1-15 means length 4-18 with 18 bytes adding an extra byte. // ONLY "Min match of 4 bytes" is supported. type LZ4sConverter struct { + noAsm bool } // ConvertBlock will convert an LZ4s block and append it as an S2 @@ -37,7 +38,7 @@ s, d := 0, len(dst) dst = dst[:cap(dst)] - if !debug && hasAmd64Asm { + if !debug && hasAsm && !l.noAsm { res, sz := cvtLZ4sBlockAsm(dst[d:], src) if res < 0 { const ( @@ -101,7 +102,8 @@ } s++ if ll > 0 { - if d+ll > dLimit { + // Leave room for a 5-byte literal header and an 8-byte copy. + if d+ll > len(dst)-13 { return nil, 0, ErrDstTooSmall } if debug { @@ -218,11 +220,13 @@ dst[2] = uint8(length >> 0) dst[1] = 0 dst[0] = 7<<2 | tagCopy1 + d += 5 if left > 0 { - d += 5 + emitRepeat16(dst[5:], offset, left) - break + // The loop condition checks room for the next chunk. + dst = dst[5:] + length = left + continue } - d += 5 break } } @@ -297,7 +301,7 @@ s, d := 0, len(dst) dst = dst[:cap(dst)] // Use assembly when possible - if !debug && hasAmd64Asm { + if !debug && hasAsm && !l.noAsm { res, sz := cvtLZ4sBlockSnappyAsm(dst[d:], src) if res < 0 { const ( --- a/vendor/github.com/klauspost/compress/s2/reader.go +++ b/vendor/github.com/klauspost/compress/s2/reader.go @@ -216,13 +216,18 @@ return r.err == nil } if rs, ok := r.r.(io.ReadSeeker); ok { - _, err := rs.Seek(int64(n), io.SeekCurrent) - if err == nil { - return true - } - if err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) { - r.err = ErrCorrupt - return false + if cur, err := rs.Seek(0, io.SeekCurrent); err == nil { + if end, err := rs.Seek(0, io.SeekEnd); err == nil { + if cur+int64(n) <= end { + if _, err := rs.Seek(cur+int64(n), io.SeekStart); err == nil { + return true + } + } + if _, err := rs.Seek(cur, io.SeekStart); err != nil { + r.err = ErrCorrupt + return false + } + } } } for n > 0 { @@ -446,9 +451,7 @@ } // Writer var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { for toWrite := range queue { entry := <-toWrite reUse <- toWrite @@ -475,7 +478,7 @@ } aWritten += int64(n) } - }() + }) defer func() { if r.err != nil { --- a/vendor/github.com/klauspost/compress/s2/writer.go +++ b/vendor/github.com/klauspost/compress/s2/writer.go @@ -139,11 +139,9 @@ toWrite := make(chan chan result, w.concurrency) w.output = toWrite - w.writerWg.Add(1) // Start a writer goroutine that will write all output in order. - go func() { - defer w.writerWg.Done() + w.writerWg.Go(func() { // Get a queued write. for write := range toWrite { @@ -175,7 +173,7 @@ // This can be used for synchronizing flushes. close(write) } - }() + }) } // Write satisfies the io.Writer interface. @@ -811,7 +809,7 @@ if w.pad <= 1 { compSize = w.written } - index = w.index.appendTo(w.ibuf[:0], w.uncompWritten, compSize) + index = w.index.appendTo(w.ibuf[:0], compSize, w.uncompWritten) // Count as written for padding. if w.appendIndex { w.written += int64(len(index)) --- a/vendor/github.com/klauspost/compress/zstd/README.md +++ b/vendor/github.com/klauspost/compress/zstd/README.md @@ -75,14 +75,47 @@ To reuse the encoder, you can use the `Reset(io.Writer)` function to change to another output. This will allow the encoder to reuse all resources and avoid wasteful allocations. -Currently stream encoding has 'light' concurrency, meaning up to 2 goroutines can be working on part -of a stream. This is independent of the `WithEncoderConcurrency(n)`, but that is likely to change +By default, stream encoding has 'light' concurrency, meaning up to 2 goroutines can be working on part +of a stream. This is independent of the `WithEncoderConcurrency(n)`, but that is likely to change in the future. So if you want to limit concurrency for future updates, specify the concurrency you would like. If you would like stream encoding to be done without spawning async goroutines, use `WithEncoderConcurrency(1)` which will compress input as each block is completed, blocking on writes until each has completed. +#### Parallel Stream Compression + +For maximum throughput on large streams, use `WithConcurrentBlocks(true)` together with +`WithEncoderConcurrency(n)` where n is the number of CPU cores you want to use. +This splits the input into large sections (jobs) that are compressed simultaneously by multiple goroutines, +similar to how the C zstd library does multithreaded compression. + +```Go +enc, err := zstd.NewWriter(out, + zstd.WithEncoderLevel(zstd.SpeedDefault), + zstd.WithEncoderConcurrency(runtime.GOMAXPROCS(0)), + zstd.WithConcurrentBlocks(true), +) +``` + +Each non-first job receives an overlap prefix from the previous job for match context, +so compression ratio is only marginally affected. Output is flushed in order, +producing a valid single-frame zstd stream. + +Benchmark on 1.8GB GOB stream (AMD Ryzen 9 9950X): + +| Level | 1 thread | 4 threads | 16 threads | 1T ratio | 16T ratio | +|---------|:----------:|:------------------:|:-------------------:|:--------:|:---------:| +| fastest | 783 MB/s | 2950 MB/s (3.8×) | 6939 MB/s (8.9×) | 12.24% | 12.26% | +| default | 728 MB/s | 2533 MB/s (3.5×) | 5340 MB/s (7.3×) | 10.67% | 10.68% | +| better | 434 MB/s | 1105 MB/s (2.5×) | 2206 MB/s (5.1×) | 9.14% | 9.21% | +| best | 129 MB/s | 367 MB/s (2.8×) | 884 MB/s (6.8×) | 8.48% | 8.63% | + +Notes: +* Not compatible with dictionary encoding. +* `Flush()` dispatches the current partial job, so latency-sensitive callers can force output. +* `EncodeAll` is unaffected — it uses its own concurrency via the encoder pool. + You can specify your desired compression level using `WithEncoderLevel()` option. Currently only pre-defined compression settings can be specified. --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -240,9 +240,11 @@ b.dst[i] = v } hist.appendKeep(b.dst) + hist.decoders.consumeSyncLen(len(b.dst)) return nil case blockTypeRaw: hist.appendKeep(b.data) + hist.decoders.consumeSyncLen(len(b.data)) return nil case blockTypeCompressed: saved := b.dst @@ -400,8 +402,9 @@ } } var err error - // Use our out buffer. - huff.MaxDecodedSize = litRegenSize + // Decoder.Decompress* uses cap(dst) for the size limit. Do not write + // MaxDecodedSize on hist.huffTree: with a trained dictionary that + // pointer aliases the shared dict.litEnc and concurrent DecodeAll races. if fourStreams { literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) } else { @@ -486,6 +489,7 @@ } if hist.decoders.nSeqs == 0 { b.dst = append(b.dst, hist.decoders.literals...) + hist.decoders.consumeSyncLen(len(hist.decoders.literals)) return nil } before := len(hist.decoders.out) @@ -673,10 +677,6 @@ hbytes := hist.b if len(hbytes) > hist.windowSize { hbytes = hbytes[len(hbytes)-hist.windowSize:] - // We do not need history anymore. - if hist.dict != nil { - hist.dict.content = nil - } } hist.decoders.windowSize = hist.windowSize hist.decoders.out = b.dst[:0] --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -7,8 +7,11 @@ import ( "context" "encoding/binary" + "errors" "io" + "maps" "sync" + "sync/atomic" "github.com/klauspost/compress/zstd/internal/xxhash" ) @@ -20,7 +23,8 @@ // A decoder can safely be re-used even if the previous stream failed. // To release the resources, you must call the Close() function on a decoder. type Decoder struct { - o decoderOptions + // publishedOptions contains the current immutable decoder options. + publishedOptions atomic.Pointer[decoderOptions] // Unreferenced decoders, ready for use. decoders chan *blockDec @@ -84,9 +88,10 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { initPredefined() var d Decoder - d.o.setDefault() - for _, o := range opts { - err := o(&d.o) + var o decoderOptions + o.setDefault() + for _, opt := range opts { + err := opt(&o) if err != nil { return nil, err } @@ -99,17 +104,19 @@ } // Initialize dict map if needed. - if d.o.dicts == nil { - d.o.dicts = make(map[uint32]*dict) + if o.dicts == nil { + o.dicts = make(map[uint32]*dict) } // Create decoders - d.decoders = make(chan *blockDec, d.o.concurrent) - for i := 0; i < d.o.concurrent; i++ { - dec := newBlockDec(d.o.lowMem) - dec.localFrame = newFrameDec(d.o) + d.decoders = make(chan *blockDec, o.concurrent) + for i := 0; i < o.concurrent; i++ { + dec := newBlockDec(o.lowMem) + dec.localFrame = newFrameDec(o) + dec.localFrame.o.dicts = nil d.decoders <- dec } + d.publishedOptions.Store(&o) if r == nil { return &d, nil @@ -171,6 +178,9 @@ d.drainOutput() d.syncStream.br.r = nil + if d.frame != nil { + d.frame.o.dicts = nil + } if r == nil { d.current.err = ErrDecoderNilInput if len(d.current.b) > 0 { @@ -180,8 +190,10 @@ return nil } + o := *d.publishedOptions.Load() + // If bytes buffer and < 5MB, do sync decoding anyway. - if bb, ok := r.(byter); ok && bb.Len() < d.o.decodeBufsBelow && !d.o.limitToCap { + if bb, ok := r.(byter); ok && bb.Len() < o.decodeBufsBelow && !o.limitToCap { bb2 := bb if debugDecoder { println("*bytes.Buffer detected, doing sync decode, len:", bb.Len()) @@ -193,18 +205,30 @@ } dst, err := d.DecodeAll(b, dst) - if err == nil { - err = io.EOF + if !errors.Is(err, ErrDecoderSizeExceeded) { + if err == nil { + err = io.EOF + } + // Save output buffer + d.syncStream.dstBuf = dst + d.current.b = dst + d.current.err = err + d.current.flushed = true + if debugDecoder { + println("sync decode to", len(dst), "bytes, err:", err) + } + return nil } - // Save output buffer - d.syncStream.dstBuf = dst - d.current.b = dst - d.current.err = err - d.current.flushed = true + // The output does not fit WithDecoderMaxMemory, which caps the total + // decoded size when decoding in memory. A Reader follows the + // streaming contract, where the option caps the window instead, so + // decode this input as a stream after all; the shortcut has not + // consumed the reader. Drop the oversized buffer rather than keep it + // for reuse. if debugDecoder { - println("sync decode to", len(dst), "bytes, err:", err) + println("sync decode exceeded max memory after", len(dst), "bytes; streaming instead") } - return nil + d.syncStream.dstBuf = nil } // Remove current block. d.stashDecoder() @@ -217,14 +241,16 @@ // Ensure no-one else is still running... d.streamWg.Wait() if d.frame == nil { - d.frame = newFrameDec(d.o) + d.frame = newFrameDec(o) + } else { + d.frame.setOptions(o) } - if d.o.concurrent == 1 { + if o.concurrent == 1 { return d.startSyncDecoder(r) } - d.current.output = make(chan decodeOutput, d.o.concurrent) + d.current.output = make(chan decodeOutput, o.concurrent) ctx, cancel := context.WithCancel(context.Background()) d.current.cancel = cancel d.streamWg.Add(1) @@ -238,13 +264,21 @@ // Options are applied on top of the existing options. // Some options cannot be changed on reset and will return an error. func (d *Decoder) ResetWithOptions(r io.Reader, opts ...DOption) error { - d.o.resetOpt = true - defer func() { d.o.resetOpt = false }() + if d.current.err == ErrDecoderClosed { + return d.current.err + } + next := *d.publishedOptions.Load() + next.dicts = maps.Clone(next.dicts) + next.resetOpt = true for _, o := range opts { - if err := o(&d.o); err != nil { + if err := o(&next); err != nil { + next.resetOpt = false + d.publishedOptions.Store(&next) return err } } + next.resetOpt = false + d.publishedOptions.Store(&next) return d.Reset(r) } @@ -324,6 +358,8 @@ // Grab a block decoder and frame decoder. block := <-d.decoders frame := block.localFrame + o := *d.publishedOptions.Load() + frame.setOptions(o) initialSize := len(dst) defer func() { if debugDecoder { @@ -335,6 +371,7 @@ frame.history.decoders.br.in = nil frame.history.decoders.br.cursor = 0 } + frame.o.dicts = nil d.decoders <- block }() frame.bBuf = input @@ -354,20 +391,20 @@ if err = d.setDict(frame); err != nil { return nil, err } - if frame.WindowSize > d.o.maxWindowSize { + if frame.WindowSize > frame.o.maxWindowSize { if debugDecoder { - println("window size exceeded:", frame.WindowSize, ">", d.o.maxWindowSize) + println("window size exceeded:", frame.WindowSize, ">", frame.o.maxWindowSize) } return dst, ErrWindowSizeExceeded } if frame.FrameContentSize != fcsUnknown { - if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)-initialSize) { + if frame.FrameContentSize > frame.o.maxDecodedSize-uint64(len(dst)-initialSize) { if debugDecoder { - println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", d.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst)) + println("decoder size exceeded; fcs:", frame.FrameContentSize, "> mcs:", frame.o.maxDecodedSize-uint64(len(dst)-initialSize), "len:", len(dst)) } return dst, ErrDecoderSizeExceeded } - if d.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) { + if frame.o.limitToCap && frame.FrameContentSize > uint64(cap(dst)-len(dst)) { if debugDecoder { println("decoder size exceeded; fcs:", frame.FrameContentSize, "> (cap-len)", cap(dst)-len(dst)) } @@ -380,14 +417,14 @@ } } - if cap(dst) == 0 && !d.o.limitToCap { + if cap(dst) == 0 && !frame.o.limitToCap { // Allocate len(input) * 2 by default if nothing is provided // and we didn't get frame content size. size := min( // Cap to 1 MB. len(input)*2, 1<<20) - if uint64(size) > d.o.maxDecodedSize { - size = int(d.o.maxDecodedSize) + if uint64(size) > frame.o.maxDecodedSize { + size = int(frame.o.maxDecodedSize) } dst = make([]byte, 0, size) } @@ -396,7 +433,7 @@ if err != nil { return dst, err } - if uint64(len(dst)-initialSize) > d.o.maxDecodedSize { + if uint64(len(dst)-initialSize) > frame.o.maxDecodedSize { return dst, ErrDecoderSizeExceeded } if len(frame.bBuf) == 0 { @@ -459,7 +496,7 @@ println("got", len(d.current.b), "bytes, error:", d.current.err, "data crc:", tmp) } - if d.o.ignoreChecksum { + if d.frame.o.ignoreChecksum { return true } @@ -497,7 +534,7 @@ if d.current.err != nil { return false } - if d.frame.WindowSize > d.o.maxDecodedSize || d.frame.WindowSize > d.o.maxWindowSize { + if d.frame.WindowSize > d.frame.o.maxDecodedSize || d.frame.WindowSize > d.frame.o.maxWindowSize { d.current.err = ErrDecoderSizeExceeded return false } @@ -546,11 +583,11 @@ // Update/Check CRC if d.frame.HasCheckSum { - if !d.o.ignoreChecksum { + if !d.frame.o.ignoreChecksum { d.frame.crc.Write(d.current.b) } if d.current.d.Last { - if !d.o.ignoreChecksum { + if !d.frame.o.ignoreChecksum { d.current.err = d.frame.checkCRC() } else { d.current.err = d.frame.consumeCRC() @@ -656,8 +693,8 @@ defer d.streamWg.Done() br := readerWrapper{r: r} - var seqDecode = make(chan *blockDec, d.o.concurrent) - var seqExecute = make(chan *blockDec, d.o.concurrent) + var seqDecode = make(chan *blockDec, d.frame.o.concurrent) + var seqExecute = make(chan *blockDec, d.frame.o.concurrent) // Async 1: Decode sequences... go func() { @@ -858,9 +895,9 @@ if err == nil { err = d.setDict(frame) } - if err == nil && d.frame.WindowSize > d.o.maxWindowSize { + if err == nil && d.frame.WindowSize > d.frame.o.maxWindowSize { if debugDecoder { - println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.o.maxWindowSize) + println("decoder size exceeded, fws:", d.frame.WindowSize, "> mws:", d.frame.o.maxWindowSize) } err = ErrDecoderSizeExceeded @@ -940,7 +977,7 @@ } func (d *Decoder) setDict(frame *frameDec) (err error) { - dict, ok := d.o.dicts[frame.DictionaryID] + dict, ok := frame.o.dicts[frame.DictionaryID] if ok { if debugDecoder { println("setting dict", frame.DictionaryID) --- a/vendor/github.com/klauspost/compress/zstd/dict.go +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -219,6 +219,9 @@ if len(contents) == 0 { return nil, errors.New("no content provided") } + if o.Level < speedNotSet || o.Level >= speedLast { + return nil, fmt.Errorf("unknown encoder level %d", o.Level) + } d := dict{ id: o.ID, litEnc: nil, @@ -230,7 +233,7 @@ } block := blockEnc{lowMem: false} block.init() - enc := encoder(&bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(maxMatchLen), bufferReset: math.MaxInt32 - int32(maxMatchLen*2), lowMem: false}}) + var enc encoder if o.Level != 0 { eOpts := encoderOptions{ level: o.Level, @@ -242,6 +245,7 @@ enc = eOpts.encoder() } else { o.Level = SpeedBestCompression + enc = encoder(&bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(maxMatchLen), bufferReset: math.MaxInt32 - int32(maxMatchLen*2), lowMem: false}}) } var ( remain [256]int @@ -295,40 +299,81 @@ if offset > 3 { newOffsets[offset-3]++ } else { - newOffsets[uint32(o.Offsets[offset-1])]++ + // Repeat codes reference the training Offsets. Skip unset + // (zero) entries so they are not ranked as real offsets. + prev := o.Offsets[offset-1] + if prev > 0 { + newOffsets[uint32(prev)]++ + } } } } // Find most used offsets. var sortedOffsets []uint32 for k := range newOffsets { + if k == 0 { + continue + } sortedOffsets = append(sortedOffsets, k) } sort.Slice(sortedOffsets, func(i, j int) bool { a, b := sortedOffsets[i], sortedOffsets[j] - if a == b { + ca, cb := newOffsets[a], newOffsets[b] + if ca == cb { // Prefer the longer offset - return sortedOffsets[i] > sortedOffsets[j] + return a > b } - return newOffsets[sortedOffsets[i]] > newOffsets[sortedOffsets[j]] + return ca > cb }) - if len(sortedOffsets) > 3 { - if debug { - print("Offsets:") - for i, v := range sortedOffsets { - if i > 20 { - break - } - printf("[%d: %d],", v, newOffsets[v]) + if debug { + print("Offsets:") + for i, v := range sortedOffsets { + if i > 20 { + break } - println("") + printf("[%d: %d],", v, newOffsets[v]) } - - sortedOffsets = sortedOffsets[:3] + println("") } - for i, v := range sortedOffsets { - o.Offsets[i] = int(v) + // Dictionary recent-offsets must be three positive values within the + // history. Ranked matches may be fewer (or empty when only unset + // repeat codes were seen), so fill remaining slots with defaults. + used := make(map[int]bool, 3) + var finalOffsets [3]int + nOff := 0 + for _, v := range sortedOffsets { + iv := int(v) + if iv <= 0 || iv > len(hist) || used[iv] { + continue + } + finalOffsets[nOff] = iv + used[iv] = true + nOff++ + if nOff == 3 { + break + } } + for _, def := range []int{1, 4, 8} { + if nOff == 3 { + break + } + if def <= len(hist) && !used[def] { + finalOffsets[nOff] = def + used[def] = true + nOff++ + } + } + for def := 1; nOff < 3 && def <= len(hist); def++ { + if !used[def] { + finalOffsets[nOff] = def + used[def] = true + nOff++ + } + } + if nOff < 3 { + return nil, fmt.Errorf("could not determine 3 valid dictionary offsets (history size %d)", len(hist)) + } + o.Offsets = finalOffsets if debug { println("New repeat offsets", o.Offsets) } @@ -336,6 +381,9 @@ if nUsed == 0 || seqs == 0 { return nil, fmt.Errorf("%d blocks, %d sequences found", nUsed, seqs) } + if litTotal == 0 { + return nil, errors.New("0 literals found") + } if debug { println("Sequences:", seqs, "Blocks:", nUsed, "Literals:", litTotal) } @@ -516,11 +564,10 @@ out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[1]))) out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[2]))) out.Write(hist) + if _, err := loadDict(out.Bytes()); err != nil { + return nil, fmt.Errorf("built dictionary failed validation: %w", err) + } if debug { - _, err := loadDict(out.Bytes()) - if err != nil { - panic(err) - } i, err := InspectDictionary(out.Bytes()) if err != nil { panic(err) --- a/vendor/github.com/klauspost/compress/zstd/enc_base.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_base.go @@ -128,6 +128,34 @@ return int32(matchLen(src[s:], src[t:])) } +// resetBasePrefix resets the encoder state and loads prefix as initial history. +// This is used for parallel job encoding where non-first jobs need overlap context. +// Rep offsets are set to defaults [1,4,8] (invalidated, matching C behavior). +func (e *fastBase) resetBasePrefix(prefix []byte) { + if e.blk == nil { + e.blk = &blockEnc{lowMem: e.lowMem} + e.blk.init() + } else { + e.blk.reset(nil) + } + e.blk.initNewEncode() + if e.crc == nil { + e.crc = xxhash.New() + } else { + e.crc.Reset() + } + e.blk.dictLitEnc = nil + e.ensureHist(len(prefix) + maxCompressedBlockSize) + // Bump cur so old table entries fall outside the window. + // When cur >= bufferReset, leave it; the first Encode call + // will shift/clear tables, preserving valid prefix entries. + if e.cur < e.bufferReset { + e.cur += e.maxMatchOff + int32(len(e.hist)) + } + e.hist = e.hist[:0] + e.hist = append(e.hist, prefix...) +} + // Reset the encoding table. func (e *fastBase) resetBase(d *dict, singleBlock bool) { if e.blk == nil { --- a/vendor/github.com/klauspost/compress/zstd/enc_best.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_best.go @@ -551,3 +551,18 @@ // Reset table to initial state copy(e.table[:], e.dictTable) } + +func (e *bestFastEncoder) ResetPrefix(prefix []byte) { + e.resetBasePrefix(prefix) + if len(prefix) < 8 { + return + } + end := e.cur + int32(len(prefix)) - 8 + for i := e.cur; i < end; i++ { + cv := load6432(prefix, i-e.cur) + h := hashLen(cv, bestLongTableBits, bestLongLen) + e.longTable[h] = prevEntry{offset: i, prev: e.longTable[h].offset} + h0 := hashLen(cv, bestShortTableBits, bestShortLen) + e.table[h0] = prevEntry{offset: i, prev: e.table[h0].offset} + } +} --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -1096,6 +1096,20 @@ } } +func (e *betterFastEncoder) ResetPrefix(prefix []byte) { + e.resetBasePrefix(prefix) + if len(prefix) < 8 { + return + } + end := e.cur + int32(len(prefix)) - 8 + for i := e.cur; i < end; i += 2 { + cv := load6432(prefix, i-e.cur) + h := hashLen(cv, betterLongTableBits, betterLongLen) + e.longTable[h] = prevEntry{offset: i, prev: e.longTable[h].offset} + e.table[hashLen(cv>>8, betterShortTableBits, betterShortLen)] = tableEntry{val: uint32(cv >> 8), offset: i + 1} + } +} + // ResetDict will reset and set a dictionary if not nil func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { e.resetBase(d, singleBlock) @@ -1229,6 +1243,10 @@ e.allDirty = false } +func (e *betterFastEncoderDict) ResetPrefix([]byte) { + panic("ResetPrefix not supported for dict encoders") +} + func (e *betterFastEncoderDict) markLongShardDirty(entryNum uint32) { e.longTableShardDirty[entryNum/betterLongTableShardSize] = true } --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -1037,6 +1037,18 @@ } } +func (e *doubleFastEncoder) ResetPrefix(prefix []byte) { + e.fastEncoder.ResetPrefix(prefix) + if len(prefix) < 8 { + return + } + end := e.cur + int32(len(prefix)) - 8 + for i := e.cur + 1; i < end; i += 2 { + cv := load6432(prefix, i-e.cur) + e.longTable[hashLen(cv, dFastLongTableBits, dFastLongLen)] = tableEntry{val: uint32(cv), offset: i} + } +} + // ResetDict will reset and set a dictionary if not nil func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { allDirty := e.allDirty @@ -1102,6 +1114,10 @@ } } +func (e *doubleFastEncoderDict) ResetPrefix([]byte) { + panic("ResetPrefix not supported for dict encoders") +} + func (e *doubleFastEncoderDict) markLongShardDirty(entryNum uint32) { e.longTableShardDirty[entryNum/dLongTableShardSize] = true } --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -797,6 +797,19 @@ } } +func (e *fastEncoder) ResetPrefix(prefix []byte) { + e.resetBasePrefix(prefix) + if len(prefix) < 8 { + return + } + end := e.cur + int32(len(prefix)) - 8 + // Index every 4th + for i := e.cur + 1; i < end; i += 4 { + cv := load6432(prefix, i-e.cur) + e.table[hashLen(cv, tableBits, tableFastHashLen)] = tableEntry{val: uint32(cv), offset: i} + } +} + // ResetDict will reset and set a dictionary if not nil func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { e.resetBase(d, singleBlock) @@ -866,6 +879,10 @@ e.allDirty = false } +func (e *fastEncoderDict) ResetPrefix([]byte) { + panic("ResetPrefix not supported for dict encoders") +} + func (e *fastEncoderDict) markAllShardsDirty() { e.allDirty = true } --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -38,6 +38,7 @@ WindowSize(size int64) int32 UseBlock(*blockEnc) Reset(d *dict, singleBlock bool) + ResetPrefix(prefix []byte) } type encoderState struct { @@ -60,6 +61,9 @@ wg sync.WaitGroup // This waitgroup indicates we have a block encoding/writing. wWg sync.WaitGroup + + // Parallel job state (used when concurrentBlocks is enabled). + jobs jobState } // NewWriter will create a new Zstandard encoder. @@ -74,6 +78,9 @@ return nil, err } } + if e.o.concurrentBlocks && (e.o.dict != nil || e.o.concurrent <= 1) { + e.o.concurrentBlocks = false + } if w != nil { e.Reset(w) } @@ -95,12 +102,31 @@ // as a new, independent stream. func (e *Encoder) Reset(w io.Writer) { s := &e.state + + if e.o.concurrentBlocks { + e.shutdownJobWorkers() + js := &s.jobs + js.jobSize = e.o.jobSize() + js.overlapSize = e.o.overlapSize() + // js.filling is allocated lazily on first Write/ReadFrom so callers + // that only use EncodeAll don't pay the (up to ~32 MB) jobSize cost. + js.filling = js.filling[:0] + if js.nextPrefix != nil { + js.putOverlapBuf(js.nextPrefix) + js.nextPrefix = nil + } + js.jobSeq = 0 + js.flushedSeq = 0 + js.flusherErr = nil + js.started = false + } + s.wg.Wait() s.wWg.Wait() if cap(s.filling) == 0 { s.filling = make([]byte, 0, e.o.blockSize) } - if e.o.concurrent > 1 { + if e.o.concurrent > 1 && !e.o.concurrentBlocks { if cap(s.current) == 0 { s.current = make([]byte, 0, e.o.blockSize) } @@ -145,6 +171,9 @@ } } hasDict := e.o.dict != nil + if e.o.concurrentBlocks && hasDict { + e.o.concurrentBlocks = false + } if hadDict != hasDict { // Dict presence changed — encoder type must be recreated. e.state.encoder = nil @@ -176,6 +205,49 @@ if s.eofWritten { return 0, ErrEncoderClosed } + if e.o.concurrentBlocks { + return e.writeJobs(p) + } + return e.writeBlocks(p) +} + +func (e *Encoder) writeJobs(p []byte) (n int, err error) { + s := &e.state + js := &s.jobs + jobSize := js.jobSize + if cap(js.filling) == 0 && len(p) > 0 { + js.filling = make([]byte, 0, jobSize) + } + for len(p) > 0 { + if len(p)+len(js.filling) < jobSize { + if e.o.crc { + _, _ = s.encoder.CRC().Write(p) + } + js.filling = append(js.filling, p...) + return n + len(p), nil + } + add := p + if len(p)+len(js.filling) > jobSize { + add = add[:jobSize-len(js.filling)] + } + if e.o.crc { + _, _ = s.encoder.CRC().Write(add) + } + js.filling = append(js.filling, add...) + p = p[len(add):] + n += len(add) + if len(js.filling) < jobSize { + return n, nil + } + if err := e.dispatchJob(false); err != nil { + return n, err + } + } + return n, nil +} + +func (e *Encoder) writeBlocks(p []byte) (n int, err error) { + s := &e.state for len(p) > 0 { if len(p)+len(s.filling) < e.o.blockSize { if e.o.crc { @@ -374,6 +446,10 @@ println("Using ReadFrom") } + if e.o.concurrentBlocks { + return e.readFromJobs(r) + } + // Flush any current writes. if len(e.state.filling) > 0 { if err := e.nextBlock(false); err != nil { @@ -387,7 +463,6 @@ if e.o.crc { _, _ = e.state.encoder.CRC().Write(src[:n2]) } - // src is now the unfilled part... src = src[n2:] n += int64(n2) switch err { @@ -420,15 +495,63 @@ } } +func (e *Encoder) readFromJobs(r io.Reader) (n int64, err error) { + js := &e.state.jobs + jobSize := js.jobSize + + // Flush any current filling. + if len(js.filling) > 0 { + if err := e.dispatchJob(false); err != nil { + return 0, err + } + } + + if cap(js.filling) < jobSize { + js.filling = make([]byte, 0, jobSize) + } + js.filling = js.filling[:jobSize] + src := js.filling + for { + n2, err := r.Read(src) + if e.o.crc { + _, _ = e.state.encoder.CRC().Write(src[:n2]) + } + src = src[n2:] + n += int64(n2) + switch err { + case io.EOF: + js.filling = js.filling[:len(js.filling)-len(src)] + return n, nil + case nil: + default: + e.state.err = err + return n, err + } + if len(src) > 0 { + continue + } + if err = e.dispatchJob(false); err != nil { + return n, err + } + if cap(js.filling) < jobSize { + js.filling = make([]byte, 0, jobSize) + } + js.filling = js.filling[:jobSize] + src = js.filling + } +} + // Flush will send the currently written data to output // and block until everything has been written. // This should only be used on rare occasions where pushing the currently queued data is critical. func (e *Encoder) Flush() error { s := &e.state + if e.o.concurrentBlocks { + return e.flushJobs() + } if len(s.filling) > 0 { err := e.nextBlock(false) if err != nil { - // Ignore Flush after Close. if errors.Is(s.err, ErrEncoderClosed) { return nil } @@ -438,7 +561,6 @@ s.wg.Wait() s.wWg.Wait() if s.err != nil { - // Ignore Flush after Close. if errors.Is(s.err, ErrEncoderClosed) { return nil } @@ -447,6 +569,20 @@ return s.writeErr } +func (e *Encoder) flushJobs() error { + js := &e.state.jobs + if len(js.filling) > 0 { + if err := e.dispatchJob(false); err != nil { + return err + } + } + e.waitAllJobs() + js.mu.Lock() + fErr := js.flusherErr + js.mu.Unlock() + return fErr +} + // Close will flush the final output and close the stream. // The function will block until everything has been written. // The Encoder can still be re-used after calling this. @@ -455,12 +591,16 @@ if s.encoder == nil { return nil } + if e.o.concurrentBlocks { + return e.closeJobs() + } if s.w == nil { if len(s.filling) == 0 && !s.headerWritten && !s.eofWritten && s.nInput == 0 { return nil } return errors.New("zstd: encoder has no writer") } + err := e.nextBlock(true) if err != nil { if errors.Is(s.err, ErrEncoderClosed) { @@ -511,6 +651,68 @@ return s.err } +func (e *Encoder) closeJobs() error { + s := &e.state + js := &s.jobs + + if errors.Is(s.err, ErrEncoderClosed) { + return nil + } + + if s.w == nil { + if len(js.filling) == 0 && !s.headerWritten && !s.eofWritten && s.nInput == 0 { + return nil + } + return errors.New("zstd: encoder has no writer") + } + + if err := e.dispatchJob(true); err != nil { + e.shutdownJobWorkers() + if errors.Is(s.err, ErrEncoderClosed) { + return nil + } + return err + } + + if s.frameContentSize > 0 && s.nInput != s.frameContentSize { + e.shutdownJobWorkers() + return fmt.Errorf("frame content size %d given, but %d bytes was written", s.frameContentSize, s.nInput) + } + + if s.fullFrameWritten { + e.shutdownJobWorkers() + s.err = ErrEncoderClosed + return nil + } + + e.shutdownJobWorkers() + if js.flusherErr != nil { + return js.flusherErr + } + + // Write CRC + if e.o.crc { + var tmp [4]byte + _, s.err = s.w.Write(s.encoder.AppendCRC(tmp[:0])) + s.nWritten += 4 + } + + // Add padding + if s.err == nil && e.o.pad > 0 { + add := calcSkippableFrame(s.nWritten, int64(e.o.pad)) + frame, err := skippableFrame(js.filling[:0], add, rand.Reader) + if err != nil { + return err + } + _, s.err = s.w.Write(frame) + } + if s.err == nil { + s.err = ErrEncoderClosed + return nil + } + return s.err +} + // EncodeAll will encode all input in src and append it to dst. // This function can be called concurrently, but each call will only run on a single goroutine. // If empty input is given, nothing is returned, unless WithZeroFrames is specified. --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -14,22 +14,23 @@ // options retains accumulated state of multiple options. type encoderOptions struct { - resetOpt bool - concurrent int - level EncoderLevel - single *bool - pad int - blockSize int - windowSize int - crc bool - fullZero bool - noEntropy bool - allLitEntropy bool - customWindow bool - customALEntropy bool - customBlockSize bool - lowMem bool - dict *dict + resetOpt bool + concurrent int + level EncoderLevel + single *bool + pad int + blockSize int + windowSize int + crc bool + fullZero bool + noEntropy bool + allLitEntropy bool + customWindow bool + customALEntropy bool + customBlockSize bool + lowMem bool + dict *dict + concurrentBlocks bool } func (o *encoderOptions) setDefault() { @@ -333,6 +334,42 @@ } } +// WithConcurrentBlocks enables job-based parallel compression for streams. +// When enabled and concurrent > 1, input is split into large sections (jobs) +// that are compressed simultaneously by multiple goroutines. +// Each non-first job receives an overlap prefix from the previous job for match context. +// Output is flushed in order, producing a valid single-frame zstd stream. +// +// Currently disabled when used with dictionary encoding. +// Cannot be changed with ResetWithOptions. +func WithConcurrentBlocks(b bool) EOption { + return func(o *encoderOptions) error { + if o.resetOpt && b != o.concurrentBlocks { + return errors.New("WithConcurrentBlocks cannot be changed on Reset") + } + o.concurrentBlocks = b + return nil + } +} + +// jobSize returns the input section size per parallel job. +func (o *encoderOptions) jobSize() int { + s := max(o.windowSize*4, 512<<10) + return s +} + +// overlapSize returns the overlap prefix size for parallel jobs. +func (o *encoderOptions) overlapSize() int { + switch o.level { + case SpeedBestCompression: + return o.windowSize / 2 + case SpeedBetterCompression: + return o.windowSize / 4 + default: + return o.windowSize / 8 + } +} + // WithEncoderDict allows to register a dictionary that will be used for the encode. // // The slice dict must be in the [dictionary format] produced by --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -49,13 +49,16 @@ ) func newFrameDec(o decoderOptions) *frameDec { + d := frameDec{} + d.setOptions(o) + return &d +} + +func (d *frameDec) setOptions(o decoderOptions) { if o.maxWindowSize > o.maxDecodedSize { o.maxWindowSize = o.maxDecodedSize } - d := frameDec{ - o: o, - } - return &d + d.o = o } // reset will read the frame header and prepare for block decoding. --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s @@ -1,4 +1,4 @@ -// Code generated by command: go run gen_fse.go -out ../fse_decoder_amd64.s -pkg=zstd. DO NOT EDIT. +// Code generated by command: go run gen_fse.go -out ../fse_decoder.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. //go:build !appengine && !noasm && gc && !noasm --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || appengine || !gc || noasm package zstd --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go @@ -52,6 +52,11 @@ d.n = 0 } +// maxAsmSize bounds the input handed to a single writeBlocks call. Assembly is +// never preemptible, so an unbounded call holds every P in stop-the-world for +// its duration. Whole 32-byte blocks, so Write can feed it in pieces. +const maxAsmSize = 128 << 10 // 4096 whole 32-byte blocks + // Size always returns 8 bytes. func (d *Digest) Size() int { return 8 } @@ -83,6 +88,13 @@ d.n = 0 } + for len(b) >= maxAsmSize { + // Assembly is not preemptible, so hashing a large buffer in one call + // blocks every stop-the-world for the whole call. Feed it in chunks. + writeBlocks(d, b[:maxAsmSize]) + b = b[maxAsmSize:] + } + if len(b) >= 32 { // One or more full blocks left. nw := writeBlocks(d, b) --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s @@ -175,7 +175,11 @@ RET // func writeBlocks(d *Digest, b []byte) int -TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 +// +// Deliberately not NOSPLIT: the stack check is the preemption point that lets a +// stop-the-world proceed between chunks. The frame must be >= abi.StackSmall +// (128), or the assembler marks this leaf NOSPLIT anyway and drops that check. +TEXT ·writeBlocks(SB), $128-40 // Load fixed primes needed for round. MOVQ ·primes+0(SB), prime1 MOVQ ·primes+8(SB), prime2 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s @@ -163,7 +163,11 @@ RET // func writeBlocks(s *Digest, b []byte) int -TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 +// +// Deliberately not NOSPLIT: the stack check is the preemption point that lets a +// stop-the-world proceed between chunks. The frame must be >= abi.StackSmall +// (128), or the assembler marks this leaf NOSPLIT anyway and drops that check. +TEXT ·writeBlocks(SB), $128-40 LDP ·primes+0(SB), (prime1, prime2) // Load state. Assume v[1-4] are stored contiguously. --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -99,6 +99,29 @@ return nil } +// consumeSyncLen accounts for n bytes of frame output produced outside +// decodeSync. maxSyncLen bounds how much the frame may still produce and +// decodeSyncSimple compares it with the output buffer's slack to decide +// whether the extended 16-byte copies are safe. Blocks with sequences +// subtracted their output; raw blocks, RLE blocks and compressed blocks +// holding only literals did not, so a frame that opened with a raw block +// overstated its remaining size for every block after it and ran the +// bounds-exact copies for the rest of the frame. +// +// Zero means "no bound" and selects the conservative paths, so a block that +// meets or exceeds the bound lands there; the frame-size checks in +// runDecoder reject the excess afterwards. +func (s *sequenceDecs) consumeSyncLen(n int) { + if s.maxSyncLen == 0 { + return + } + if uint64(n) >= s.maxSyncLen { + s.maxSyncLen = 0 + return + } + s.maxSyncLen -= uint64(n) +} + func (s *sequenceDecs) freeDecoders() { if f := s.litLengths.fse; f != nil && !f.preDefined { fseDecoderPool.Put(f) --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go @@ -3,321 +3,83 @@ package zstd import ( - "fmt" - "io" - "github.com/klauspost/compress/internal/cpuinfo" ) -type decodeSyncAsmContext struct { - llTable []decSymbol - mlTable []decSymbol - ofTable []decSymbol - llState uint64 - mlState uint64 - ofState uint64 - iteration int - litRemain int - out []byte - outPosition int - literals []byte - litPosition int - history []byte - windowSize int - ll int // set on error (not for all errors, please refer to _generate/gen.go) - ml int // set on error (not for all errors, please refer to _generate/gen.go) - mo int // set on error (not for all errors, please refer to _generate/gen.go) -} +// The shared decode/decodeSync/executeSimple wrappers and context structs live +// in seqdec_asm.go; this file only declares the amd64 asm routines and the +// dispatch helpers that pick the BMI2 / non-BMI2 (and 56-bit / safe) variant. -// sequenceDecs_decodeSync_amd64 implements the main loop of sequenceDecs.decodeSync in x86 asm. +// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. // // Please refer to seqdec_generic.go for the reference implementation. // //go:noescape -func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int +func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int -// sequenceDecs_decodeSync_bmi2 implements the main loop of sequenceDecs.decodeSync in x86 asm with BMI2 extensions. +// sequenceDecs_decode_56_amd64 implements the main loop of sequenceDecs in x86 asm. // //go:noescape -func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int +func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int -// sequenceDecs_decodeSync_safe_amd64 does the same as above, but does not write more than output buffer. +// sequenceDecs_decode_bmi2 implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. // //go:noescape -func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int +func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int -// sequenceDecs_decodeSync_safe_bmi2 does the same as above, but does not write more than output buffer. +// sequenceDecs_decode_56_bmi2 implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. // //go:noescape -func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int - -// decode sequences from the stream with the provided history but without a dictionary. -func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { - if len(s.dict) > 0 { - return false, nil - } - if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSize { - return false, nil - } - - // FIXME: Using unsafe memory copies leads to rare, random crashes - // with fuzz testing. It is therefore disabled for now. - const useSafe = true - /* - useSafe := false - if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSizeAlloc { - useSafe = true - } - if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < int(s.maxSyncLen) { - useSafe = true - } - if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { - useSafe = true - } - */ - - br := s.br - - maxBlockSize := min(s.windowSize, maxCompressedBlockSize) - - ctx := decodeSyncAsmContext{ - llTable: s.litLengths.fse.dt[:maxTablesize], - mlTable: s.matchLengths.fse.dt[:maxTablesize], - ofTable: s.offsets.fse.dt[:maxTablesize], - llState: uint64(s.litLengths.state.state), - mlState: uint64(s.matchLengths.state.state), - ofState: uint64(s.offsets.state.state), - iteration: s.nSeqs - 1, - litRemain: len(s.literals), - out: s.out, - outPosition: len(s.out), - literals: s.literals, - windowSize: s.windowSize, - history: hist, - } - - s.seqSize = 0 - startSize := len(s.out) +func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int - var errCode int +// decodeAsm runs the sequenceDecs decode loop, choosing the BMI2 / 56-bit variant. +func decodeAsm(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext, lte56bits bool) int { if cpuinfo.HasBMI2() { - if useSafe { - errCode = sequenceDecs_decodeSync_safe_bmi2(s, br, &ctx) - } else { - errCode = sequenceDecs_decodeSync_bmi2(s, br, &ctx) - } - } else { - if useSafe { - errCode = sequenceDecs_decodeSync_safe_amd64(s, br, &ctx) - } else { - errCode = sequenceDecs_decodeSync_amd64(s, br, &ctx) - } - } - switch errCode { - case noError: - break - - case errorMatchLenOfsMismatch: - return true, fmt.Errorf("zero matchoff and matchlen (%d) > 0", ctx.ml) - - case errorMatchLenTooBig: - return true, fmt.Errorf("match len (%d) bigger than max allowed length", ctx.ml) - - case errorMatchOffTooBig: - return true, fmt.Errorf("match offset (%d) bigger than current history (%d)", - ctx.mo, ctx.outPosition+len(hist)-startSize) - - case errorNotEnoughLiterals: - return true, fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", - ctx.ll, ctx.litRemain+ctx.ll) - - case errorOverread: - return true, io.ErrUnexpectedEOF - - case errorNotEnoughSpace: - size := ctx.outPosition + ctx.ll + ctx.ml - if debugDecoder { - println("msl:", s.maxSyncLen, "cap", cap(s.out), "bef:", startSize, "sz:", size-startSize, "mbs:", maxBlockSize, "outsz:", cap(s.out)-startSize) + if lte56bits { + return sequenceDecs_decode_56_bmi2(s, br, ctx) } - return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) - - default: - return true, fmt.Errorf("sequenceDecs_decode returned erroneous code %d", errCode) - } - - s.seqSize += ctx.litRemain - if s.seqSize > maxBlockSize { - return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) + return sequenceDecs_decode_bmi2(s, br, ctx) } - err := br.close() - if err != nil { - printf("Closing sequences: %v, %+v\n", err, *br) - return true, err + if lte56bits { + return sequenceDecs_decode_56_amd64(s, br, ctx) } - - s.literals = s.literals[ctx.litPosition:] - t := ctx.outPosition - s.out = s.out[:t] - - // Add final literals - s.out = append(s.out, s.literals...) - if debugDecoder { - t += len(s.literals) - if t != len(s.out) { - panic(fmt.Errorf("length mismatch, want %d, got %d", len(s.out), t)) - } - } - - return true, nil -} - -// -------------------------------------------------------------------------------- - -type decodeAsmContext struct { - llTable []decSymbol - mlTable []decSymbol - ofTable []decSymbol - llState uint64 - mlState uint64 - ofState uint64 - iteration int - seqs []seqVals - litRemain int + return sequenceDecs_decode_amd64(s, br, ctx) } -const noError = 0 - -// error reported when mo == 0 && ml > 0 -const errorMatchLenOfsMismatch = 1 - -// error reported when ml > maxMatchLen -const errorMatchLenTooBig = 2 - -// error reported when mo > available history or mo > s.windowSize -const errorMatchOffTooBig = 3 - -// error reported when the sum of literal lengths exeeceds the literal buffer size -const errorNotEnoughLiterals = 4 - -// error reported when capacity of `out` is too small -const errorNotEnoughSpace = 5 - -// error reported when bits are overread. -const errorOverread = 6 - -// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. +// sequenceDecs_decodeSync_amd64 implements the main loop of sequenceDecs.decodeSync in x86 asm. // // Please refer to seqdec_generic.go for the reference implementation. // //go:noescape -func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int +func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int -// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. -// -// Please refer to seqdec_generic.go for the reference implementation. +// sequenceDecs_decodeSync_bmi2 implements the main loop of sequenceDecs.decodeSync in x86 asm with BMI2 extensions. // //go:noescape -func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int +func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int -// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. +// sequenceDecs_decodeSync_safe_amd64 does the same as above, but does not write more than output buffer. // //go:noescape -func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int +func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int -// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions. +// sequenceDecs_decodeSync_safe_bmi2 does the same as above, but does not write more than output buffer. // //go:noescape -func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int - -// decode sequences from the stream without the provided history. -func (s *sequenceDecs) decode(seqs []seqVals) error { - br := s.br - - maxBlockSize := min(s.windowSize, maxCompressedBlockSize) - - ctx := decodeAsmContext{ - llTable: s.litLengths.fse.dt[:maxTablesize], - mlTable: s.matchLengths.fse.dt[:maxTablesize], - ofTable: s.offsets.fse.dt[:maxTablesize], - llState: uint64(s.litLengths.state.state), - mlState: uint64(s.matchLengths.state.state), - ofState: uint64(s.offsets.state.state), - seqs: seqs, - iteration: len(seqs) - 1, - litRemain: len(s.literals), - } - - if debugDecoder { - println("decode: decoding", len(seqs), "sequences", br.remain(), "bits remain on stream") - } +func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int - s.seqSize = 0 - lte56bits := s.maxBits+s.offsets.fse.actualTableLog+s.matchLengths.fse.actualTableLog+s.litLengths.fse.actualTableLog <= 56 - var errCode int +// decodeSyncAsm runs the decodeSync loop, choosing the BMI2 / safe variant. +func decodeSyncAsm(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext, safe bool) int { if cpuinfo.HasBMI2() { - if lte56bits { - errCode = sequenceDecs_decode_56_bmi2(s, br, &ctx) - } else { - errCode = sequenceDecs_decode_bmi2(s, br, &ctx) - } - } else { - if lte56bits { - errCode = sequenceDecs_decode_56_amd64(s, br, &ctx) - } else { - errCode = sequenceDecs_decode_amd64(s, br, &ctx) - } - } - if errCode != 0 { - i := len(seqs) - ctx.iteration - 1 - switch errCode { - case errorMatchLenOfsMismatch: - ml := ctx.seqs[i].ml - return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml) - - case errorMatchLenTooBig: - ml := ctx.seqs[i].ml - return fmt.Errorf("match len (%d) bigger than max allowed length", ml) - - case errorNotEnoughLiterals: - ll := ctx.seqs[i].ll - return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, ctx.litRemain+ll) - case errorOverread: - return io.ErrUnexpectedEOF + if safe { + return sequenceDecs_decodeSync_safe_bmi2(s, br, ctx) } - - return fmt.Errorf("sequenceDecs_decode_amd64 returned erroneous code %d", errCode) - } - - if ctx.litRemain < 0 { - return fmt.Errorf("literal count is too big: total available %d, total requested %d", - len(s.literals), len(s.literals)-ctx.litRemain) + return sequenceDecs_decodeSync_bmi2(s, br, ctx) } - - s.seqSize += ctx.litRemain - if s.seqSize > maxBlockSize { - return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) - } - if debugDecoder { - println("decode: ", br.remain(), "bits remain on stream. code:", errCode) - } - err := br.close() - if err != nil { - printf("Closing sequences: %v, %+v\n", err, *br) + if safe { + return sequenceDecs_decodeSync_safe_amd64(s, br, ctx) } - return err -} - -// -------------------------------------------------------------------------------- - -type executeAsmContext struct { - seqs []seqVals - seqIndex int - out []byte - history []byte - literals []byte - outPosition int - litPosition int - windowSize int + return sequenceDecs_decodeSync_amd64(s, br, ctx) } // sequenceDecs_executeSimple_amd64 implements the main loop of sequenceDecs.executeSimple in x86 asm. @@ -334,54 +96,10 @@ //go:noescape func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool -// executeSimple handles cases when dictionary is not used. -func (s *sequenceDecs) executeSimple(seqs []seqVals, hist []byte) error { - // Ensure we have enough output size... - if len(s.out)+s.seqSize+compressedBlockOverAlloc > cap(s.out) { - addBytes := s.seqSize + len(s.out) + compressedBlockOverAlloc - s.out = append(s.out, make([]byte, addBytes)...) - s.out = s.out[:len(s.out)-addBytes] +// executeSimpleAsm runs the executeSimple loop, choosing the safe variant. +func executeSimpleAsm(ctx *executeAsmContext, safe bool) bool { + if safe { + return sequenceDecs_executeSimple_safe_amd64(ctx) } - - if debugDecoder { - printf("Execute %d seqs with literals: %d into %d bytes\n", len(seqs), len(s.literals), s.seqSize) - } - - var t = len(s.out) - out := s.out[:t+s.seqSize] - - ctx := executeAsmContext{ - seqs: seqs, - seqIndex: 0, - out: out, - history: hist, - outPosition: t, - litPosition: 0, - literals: s.literals, - windowSize: s.windowSize, - } - var ok bool - if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { - ok = sequenceDecs_executeSimple_safe_amd64(&ctx) - } else { - ok = sequenceDecs_executeSimple_amd64(&ctx) - } - if !ok { - return fmt.Errorf("match offset (%d) bigger than current history (%d)", - seqs[ctx.seqIndex].mo, ctx.outPosition+len(hist)) - } - s.literals = s.literals[ctx.litPosition:] - t = ctx.outPosition - - // Add final literals - copy(out[t:], s.literals) - if debugDecoder { - t += len(s.literals) - if t != len(out) { - panic(fmt.Errorf("length mismatch, want %d, got %d, ss: %d", len(out), t, s.seqSize)) - } - } - s.out = out - - return nil + return sequenceDecs_executeSimple_amd64(ctx) } --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -1,10 +1,10 @@ -// Code generated by command: go run gen.go -out ../seqdec_amd64.s -pkg=zstd. DO NOT EDIT. +// Code generated by command: go run gen.go -out ../seqdec.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT. //go:build !appengine && !noasm && gc && !noasm // func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV -TEXT ·sequenceDecs_decode_amd64(SB), $8-32 +TEXT ·sequenceDecs_decode_amd64(SB), $40-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -16,6 +16,14 @@ MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 + MOVQ (AX), CX + MOVQ CX, 8(SP) + MOVQ 24(AX), CX + MOVQ CX, 16(SP) + MOVQ 48(AX), CX + MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) MOVQ 104(AX), R10 MOVQ s+0(FP), AX MOVQ 144(AX), R11 @@ -55,46 +63,32 @@ sequenceDecs_decode_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_of_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_amd64_of_update_zero: - MOVQ AX, 16(R10) + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, 16(R10) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_amd64_ml_update_zero: - MOVQ AX, 8(R10) + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, 8(R10) // Fill bitreader to have enough for the remaining CMPQ SI, $0x08 @@ -126,90 +120,76 @@ sequenceDecs_decode_amd64_fill_2_end: // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_amd64_ll_update_zero: - MOVQ AX, (R10) + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, (R10) // Fill bitreader for state updates MOVQ R14, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_amd64_skip_update // Update Literal Length State MOVBQZX DI, R14 SHRL $0x10, DI - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R14 SHRL $0x10, R8 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R14 SHRL $0x10, R9 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decode_amd64_skip_update: @@ -290,8 +270,7 @@ sequenceDecs_decode_amd64_match_len_ofs_ok: ADDQ $0x18, R10 - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decode_amd64_main_loop MOVQ s+0(FP), AX MOVQ R11, 144(AX) @@ -303,36 +282,54 @@ MOVQ SI, 32(AX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_amd64_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_amd64_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: CMOV -TEXT ·sequenceDecs_decode_56_amd64(SB), $8-32 +TEXT ·sequenceDecs_decode_56_amd64(SB), $40-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -344,6 +341,14 @@ MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 + MOVQ (AX), CX + MOVQ CX, 8(SP) + MOVQ 24(AX), CX + MOVQ CX, 16(SP) + MOVQ 48(AX), CX + MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) MOVQ 104(AX), R10 MOVQ s+0(FP), AX MOVQ 144(AX), R11 @@ -383,132 +388,104 @@ sequenceDecs_decode_56_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_of_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_56_amd64_of_update_zero: - MOVQ AX, 16(R10) + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, 16(R10) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_56_amd64_ml_update_zero: - MOVQ AX, 8(R10) + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, 8(R10) // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R15 - SHLQ CL, R15 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decode_56_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decode_56_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decode_56_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R15 - ADDQ R15, AX - -sequenceDecs_decode_56_amd64_ll_update_zero: - MOVQ AX, (R10) + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R15 + ADDQ R15, AX + MOVQ AX, (R10) // Fill bitreader for state updates MOVQ R14, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_56_amd64_skip_update // Update Literal Length State MOVBQZX DI, R14 SHRL $0x10, DI - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R14 SHRL $0x10, R8 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R14 SHRL $0x10, R9 - LEAQ (BX)(R14*1), CX + MOVQ BX, CX MOVQ DX, R15 - MOVQ CX, BX - ROLQ CL, R15 - MOVL $0x00000001, BP - MOVB R14, CL - SHLL CL, BP - DECL BP - ANDQ BP, R15 + SHLQ CL, R15 + SHRQ $0x01, R15 + ADDQ R14, BX + XORQ $0x3f, R14 + MOVQ R14, CX + SHRQ CL, R15 ADDQ R15, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decode_56_amd64_skip_update: @@ -589,8 +566,7 @@ sequenceDecs_decode_56_amd64_match_len_ofs_ok: ADDQ $0x18, R10 - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decode_56_amd64_main_loop MOVQ s+0(FP), AX MOVQ R11, 144(AX) @@ -602,36 +578,54 @@ MOVQ SI, 32(AX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_56_amd64_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_56_amd64_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: BMI, BMI2, CMOV -TEXT ·sequenceDecs_decode_bmi2(SB), $8-32 +TEXT ·sequenceDecs_decode_bmi2(SB), $40-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -643,6 +637,14 @@ MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 + MOVQ (CX), R9 + MOVQ R9, 8(SP) + MOVQ 24(CX), R9 + MOVQ R9, 16(SP) + MOVQ 48(CX), R9 + MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) MOVQ 104(CX), R9 MOVQ s+0(FP), CX MOVQ 144(CX), R10 @@ -753,8 +755,7 @@ MOVQ R13, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R13 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_bmi2_skip_update LEAQ (SI)(DI*1), R14 ADDQ R8, R14 @@ -772,8 +773,7 @@ ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -783,8 +783,7 @@ ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -793,8 +792,7 @@ ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decode_bmi2_skip_update: @@ -875,8 +873,7 @@ sequenceDecs_decode_bmi2_match_len_ofs_ok: ADDQ $0x18, R9 - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decode_bmi2_main_loop MOVQ s+0(FP), CX MOVQ R10, 144(CX) @@ -888,36 +885,54 @@ MOVQ BX, 32(CX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_bmi2_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_bmi2_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // Requires: BMI, BMI2, CMOV -TEXT ·sequenceDecs_decode_56_bmi2(SB), $8-32 +TEXT ·sequenceDecs_decode_56_bmi2(SB), $40-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -929,6 +944,14 @@ MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 + MOVQ (CX), R9 + MOVQ R9, 8(SP) + MOVQ 24(CX), R9 + MOVQ R9, 16(SP) + MOVQ 48(CX), R9 + MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) MOVQ 104(CX), R9 MOVQ s+0(FP), CX MOVQ 144(CX), R10 @@ -1010,8 +1033,7 @@ MOVQ R13, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R13 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decode_56_bmi2_skip_update LEAQ (SI)(DI*1), R14 ADDQ R8, R14 @@ -1029,8 +1051,7 @@ ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -1040,8 +1061,7 @@ ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -1050,8 +1070,7 @@ ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decode_56_bmi2_skip_update: @@ -1132,8 +1151,7 @@ sequenceDecs_decode_56_bmi2_match_len_ofs_ok: ADDQ $0x18, R9 - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decode_56_bmi2_main_loop MOVQ s+0(FP), CX MOVQ R10, 144(CX) @@ -1145,30 +1163,48 @@ MOVQ BX, 32(CX) // Return success + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000000, ret+24(FP) RET // Return with match length error sequenceDecs_decode_56_bmi2_error_match_len_ofs_mismatch: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000001, ret+24(FP) RET // Return with match too long error sequenceDecs_decode_56_bmi2_error_match_len_too_big: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000002, ret+24(FP) RET // Return with match offset too long error + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000003, ret+24(FP) RET // Return with not enough literals error error_not_enough_literals: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000004, ret+24(FP) RET // Return with overread error error_overread: + MOVQ 32(SP), AX + MOVQ ctx+16(FP), CX + MOVQ AX, 96(CX) MOVQ $0x00000006, ret+24(FP) RET @@ -1203,9 +1239,11 @@ MOVQ 8(AX), R13 // Copy literals - TESTQ R11, R11 - JZ check_offset - XORQ R14, R14 + MOVUPS (SI), X0 + MOVUPS X0, (BX) + CMPQ R11, $0x10 + JBE copy_1_end + MOVQ $0x00000010, R14 copy_1: MOVUPS (SI)(R14*1), X0 @@ -1213,12 +1251,13 @@ ADDQ $0x10, R14 CMPQ R14, R11 JB copy_1 - ADDQ R11, SI - ADDQ R11, BX - ADDQ R11, DI + +copy_1_end: + ADDQ R11, SI + ADDQ R11, BX + ADDQ R11, DI // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: LEAQ (DI)(R10*1), R11 CMPQ R12, R11 JG error_match_off_too_big @@ -1258,13 +1297,13 @@ JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), R11 - MOVB 2(R14), R12 - MOVW R11, (BX) - MOVB R12, 2(BX) - ADDQ R13, R14 - ADDQ R13, BX - JMP copy_4_end + MOVWQZX (R14), R11 + MOVB 2(R14), R12 + MOVW R11, (BX) + MOVB R12, 2(BX) + ADDQ R13, R14 + ADDQ R13, BX + JMP copy_4_end copy_4_move_4through7: MOVL (R14), R11 @@ -1327,13 +1366,13 @@ JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (BX) - MOVB BP, 2(BX) - ADDQ R11, R14 - ADDQ R11, BX - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (BX) + MOVB BP, 2(BX) + ADDQ R11, R14 + ADDQ R11, BX + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 @@ -1366,9 +1405,19 @@ JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, DI - MOVQ BX, R12 + ADDQ R13, DI + MOVUPS (R11), X0 + MOVUPS X0, (BX) + CMPQ R13, $0x10 + JA copy_2_long + ADDQ R13, BX + JMP handle_loop + +copy_2_long: + LEAQ 16(BX), R12 ADDQ R13, BX + ADDQ $0x10, R11 + SUBQ $0x10, R13 copy_2: MOVUPS (R11), X0 @@ -1384,12 +1433,12 @@ ADDQ R13, DI copy_slow_3: - MOVB (R11), R12 - MOVB R12, (BX) - INCQ R11 - INCQ BX - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (R11), R12 + MOVB R12, (BX) + INCQ R11 + INCQ BX + DECQ R13 + JNZ copy_slow_3 handle_loop: ADDQ $0x18, AX @@ -1494,13 +1543,13 @@ JMP copy_1_end copy_1_move_3: - MOVW (SI), R14 - MOVB 2(SI), R15 - MOVW R14, (BX) - MOVB R15, 2(BX) - ADDQ R11, SI - ADDQ R11, BX - JMP copy_1_end + MOVWQZX (SI), R14 + MOVB 2(SI), R15 + MOVW R14, (BX) + MOVB R15, 2(BX) + ADDQ R11, SI + ADDQ R11, BX + JMP copy_1_end copy_1_move_4through7: MOVL (SI), R14 @@ -1563,13 +1612,13 @@ JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), R11 - MOVB 2(R14), R12 - MOVW R11, (BX) - MOVB R12, 2(BX) - ADDQ R13, R14 - ADDQ R13, BX - JMP copy_4_end + MOVWQZX (R14), R11 + MOVB 2(R14), R12 + MOVW R11, (BX) + MOVB R12, 2(BX) + ADDQ R13, R14 + ADDQ R13, BX + JMP copy_4_end copy_4_move_4through7: MOVL (R14), R11 @@ -1632,13 +1681,13 @@ JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (BX) - MOVB BP, 2(BX) - ADDQ R11, R14 - ADDQ R11, BX - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (BX) + MOVB BP, 2(BX) + ADDQ R11, R14 + ADDQ R11, BX + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 @@ -1707,13 +1756,13 @@ JMP copy_2_end copy_2_move_3: - MOVW (R11), R12 - MOVB 2(R11), R14 - MOVW R12, (BX) - MOVB R14, 2(BX) - ADDQ R13, R11 - ADDQ R13, BX - JMP copy_2_end + MOVWQZX (R11), R12 + MOVB 2(R11), R14 + MOVW R12, (BX) + MOVB R14, 2(BX) + ADDQ R13, R11 + ADDQ R13, BX + JMP copy_2_end copy_2_move_4through7: MOVL (R11), R12 @@ -1740,12 +1789,12 @@ ADDQ R13, DI copy_slow_3: - MOVB (R11), R12 - MOVB R12, (BX) - INCQ R11 - INCQ BX - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (R11), R12 + MOVB R12, (BX) + INCQ R11 + INCQ BX + DECQ R13 + JNZ copy_slow_3 handle_loop: ADDQ $0x18, AX @@ -1784,7 +1833,7 @@ // func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_amd64(SB), $96-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -1796,26 +1845,34 @@ MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 - XORQ CX, CX + MOVQ (AX), CX MOVQ CX, 8(SP) + MOVQ 24(AX), CX MOVQ CX, 16(SP) + MOVQ 48(AX), CX MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) + XORQ CX, CX + MOVQ CX, 40(SP) + MOVQ CX, 48(SP) + MOVQ CX, 56(SP) MOVQ 112(AX), R10 MOVQ 128(AX), CX - MOVQ CX, 32(SP) + MOVQ CX, 64(SP) MOVQ 144(AX), R11 MOVQ 136(AX), R12 MOVQ 200(AX), CX - MOVQ CX, 56(SP) + MOVQ CX, 88(SP) MOVQ 176(AX), CX - MOVQ CX, 48(SP) + MOVQ CX, 80(SP) MOVQ 184(AX), AX - MOVQ AX, 40(SP) - MOVQ 40(SP), AX - ADDQ AX, 48(SP) + MOVQ AX, 72(SP) + MOVQ 72(SP), AX + ADDQ AX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R10, 32(SP) + ADDQ R10, 64(SP) // outBase += outPosition ADDQ R12, R10 @@ -1853,46 +1910,32 @@ sequenceDecs_decodeSync_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_of_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_of_update_zero: - MOVQ AX, 8(SP) + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 40(SP) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_ml_update_zero: - MOVQ AX, 16(SP) + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ SI, $0x08 @@ -1924,96 +1967,82 @@ sequenceDecs_decodeSync_amd64_fill_2_end: // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_amd64_ll_update_zero: - MOVQ AX, 24(SP) + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 56(SP) // Fill bitreader for state updates MOVQ R13, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_amd64_skip_update // Update Literal Length State MOVBQZX DI, R13 SHRL $0x10, DI - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R13 SHRL $0x10, R8 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R13 SHRL $0x10, R9 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decodeSync_amd64_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ AX, $0x01 JBE sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -2022,7 +2051,7 @@ JMP sequenceDecs_decodeSync_amd64_after_adjust sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_amd64_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_amd64_adjust_offset_nonzero @@ -2057,14 +2086,11 @@ MOVQ R14, R13 sequenceDecs_decodeSync_amd64_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), AX - MOVQ 24(SP), CX - LEAQ (AX)(CX*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), AX + MOVQ 56(SP), CX MOVQ ctx+16(FP), R14 SUBQ CX, 104(R14) JS error_not_enough_literals @@ -2076,50 +2102,49 @@ JNZ sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_amd64_match_len_ofs_ok: - MOVQ 24(SP), AX - MOVQ 8(SP), CX - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ (AX)(R13*1), R14 + LEAQ 16(CX)(AX*1), R14 ADDQ R10, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ AX, AX - JZ check_offset - XORQ R14, R14 + MOVUPS (R11), X0 + MOVUPS X0, (R10) + CMPQ CX, $0x10 + JBE copy_1_end + MOVQ $0x00000010, R14 copy_1: MOVUPS (R11)(R14*1), X0 MOVUPS X0, (R10)(R14*1) ADDQ $0x10, R14 - CMPQ R14, AX + CMPQ R14, CX JB copy_1 - ADDQ AX, R11 - ADDQ AX, R10 - ADDQ AX, R12 + +copy_1_end: + ADDQ CX, R11 + ADDQ CX, R10 + ADDQ CX, R12 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVQ R12, AX - ADDQ 40(SP), AX - CMPQ CX, AX + MOVQ R12, CX + ADDQ 72(SP), CX + CMPQ R13, CX JG error_match_off_too_big - CMPQ CX, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ CX, AX - SUBQ R12, AX + MOVQ R13, CX + SUBQ R12, CX JLS copy_match - MOVQ 48(SP), R14 - SUBQ AX, R14 - CMPQ R13, AX + MOVQ 80(SP), R14 + SUBQ CX, R14 + CMPQ AX, CX JG copy_all_from_history - MOVQ R13, AX - SUBQ $0x10, AX + MOVQ AX, CX + SUBQ $0x10, CX JB copy_4_small copy_4_loop: @@ -2127,54 +2152,54 @@ MOVUPS X0, (R10) ADDQ $0x10, R14 ADDQ $0x10, R10 - SUBQ $0x10, AX + SUBQ $0x10, CX JAE copy_4_loop - LEAQ 16(R14)(AX*1), R14 - LEAQ 16(R10)(AX*1), R10 + LEAQ 16(R14)(CX*1), R14 + LEAQ 16(R10)(CX*1), R10 MOVUPS -16(R14), X0 MOVUPS X0, -16(R10) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ AX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ AX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), AX - MOVB 2(R14), CL - MOVW AX, (R10) - MOVB CL, 2(R10) - ADDQ R13, R14 - ADDQ R13, R10 - JMP copy_4_end + MOVWQZX (R14), CX + MOVB 2(R14), R13 + MOVW CX, (R10) + MOVB R13, 2(R10) + ADDQ AX, R14 + ADDQ AX, R10 + JMP copy_4_end copy_4_move_4through7: - MOVL (R14), AX - MOVL -4(R14)(R13*1), CX - MOVL AX, (R10) - MOVL CX, -4(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVL (R14), CX + MOVL -4(R14)(AX*1), R13 + MOVL CX, (R10) + MOVL R13, -4(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), AX - MOVQ -8(R14)(R13*1), CX - MOVQ AX, (R10) - MOVQ CX, -8(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVQ (R14), CX + MOVQ -8(R14)(AX*1), R13 + MOVQ CX, (R10) + MOVQ R13, -8(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 copy_4_end: - ADDQ R13, R12 + ADDQ AX, R12 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ AX, R15 + MOVQ CX, R15 SUBQ $0x10, R15 JB copy_5_small @@ -2192,90 +2217,99 @@ JMP copy_5_end copy_5_small: - CMPQ AX, $0x03 + CMPQ CX, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ AX, $0x08 + CMPQ CX, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(AX*1), BP + MOVB -1(R14)(CX*1), BP MOVB R15, (R10) - MOVB BP, -1(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVB BP, -1(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R10) - MOVB BP, 2(R10) - ADDQ AX, R14 - ADDQ AX, R10 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R10) + MOVB BP, 2(R10) + ADDQ CX, R14 + ADDQ CX, R10 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(AX*1), BP + MOVL -4(R14)(CX*1), BP MOVL R15, (R10) - MOVL BP, -4(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVL BP, -4(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(AX*1), BP + MOVQ -8(R14)(CX*1), BP MOVQ R15, (R10) - MOVQ BP, -8(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVQ BP, -8(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 copy_5_end: - ADDQ AX, R12 - SUBQ AX, R13 + ADDQ CX, R12 + SUBQ CX, AX // Copy match from the current buffer copy_match: - MOVQ R10, AX - SUBQ CX, AX + MOVQ R10, CX + SUBQ R13, CX // ml <= mo - CMPQ R13, CX + CMPQ AX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R12 - MOVQ R10, CX - ADDQ R13, R10 + ADDQ AX, R12 + MOVUPS (CX), X0 + MOVUPS X0, (R10) + CMPQ AX, $0x10 + JA copy_2_long + ADDQ AX, R10 + JMP handle_loop + +copy_2_long: + LEAQ 16(R10), R13 + ADDQ AX, R10 + ADDQ $0x10, CX + SUBQ $0x10, AX copy_2: - MOVUPS (AX), X0 - MOVUPS X0, (CX) - ADDQ $0x10, AX + MOVUPS (CX), X0 + MOVUPS X0, (R13) ADDQ $0x10, CX - SUBQ $0x10, R13 + ADDQ $0x10, R13 + SUBQ $0x10, AX JHI copy_2 JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R12 + ADDQ AX, R12 copy_slow_3: - MOVB (AX), CL - MOVB CL, (R10) - INCQ AX - INCQ R10 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (CX), R13 + MOVB R13, (R10) + INCQ CX + INCQ R10 + DECQ AX + JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_amd64_main_loop loop_finished: @@ -2284,6 +2318,13 @@ MOVB BL, 40(AX) MOVQ SI, 32(AX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R12, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R12, 136(AX) @@ -2297,7 +2338,7 @@ // Return with match length error sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -2306,7 +2347,7 @@ // Return with match too long error sequenceDecs_decodeSync_amd64_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -2314,7 +2355,7 @@ // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R12, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -2323,7 +2364,7 @@ // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -2336,9 +2377,9 @@ // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R12, 136(AX) MOVQ $0x00000005, ret+24(FP) @@ -2346,7 +2387,7 @@ // func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: BMI, BMI2, CMOV, SSE -TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_bmi2(SB), $96-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -2358,26 +2399,34 @@ MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 - XORQ R9, R9 + MOVQ (CX), R9 MOVQ R9, 8(SP) + MOVQ 24(CX), R9 MOVQ R9, 16(SP) + MOVQ 48(CX), R9 MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) + XORQ R9, R9 + MOVQ R9, 40(SP) + MOVQ R9, 48(SP) + MOVQ R9, 56(SP) MOVQ 112(CX), R9 MOVQ 128(CX), R10 - MOVQ R10, 32(SP) + MOVQ R10, 64(SP) MOVQ 144(CX), R10 MOVQ 136(CX), R11 MOVQ 200(CX), R12 - MOVQ R12, 56(SP) + MOVQ R12, 88(SP) MOVQ 176(CX), R12 - MOVQ R12, 48(SP) + MOVQ R12, 80(SP) MOVQ 184(CX), CX - MOVQ CX, 40(SP) - MOVQ 40(SP), CX - ADDQ CX, 48(SP) + MOVQ CX, 72(SP) + MOVQ 72(SP), CX + ADDQ CX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R9, 32(SP) + ADDQ R9, 64(SP) // outBase += outPosition ADDQ R11, R9 @@ -2425,7 +2474,7 @@ MOVQ R8, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 8(SP) + MOVQ CX, 40(SP) // Update match length MOVQ $0x00000808, CX @@ -2438,7 +2487,7 @@ MOVQ DI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 16(SP) + MOVQ CX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ BX, $0x08 @@ -2480,14 +2529,13 @@ MOVQ SI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 24(SP) + MOVQ CX, 56(SP) // Fill bitreader for state updates MOVQ R12, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R12 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_bmi2_skip_update LEAQ (SI)(DI*1), R13 ADDQ R8, R13 @@ -2505,8 +2553,7 @@ ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -2516,8 +2563,7 @@ ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -2526,14 +2572,13 @@ ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decodeSync_bmi2_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ R12, $0x01 JBE sequenceDecs_decodeSync_bmi2_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -2542,7 +2587,7 @@ JMP sequenceDecs_decodeSync_bmi2_after_adjust sequenceDecs_decodeSync_bmi2_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_bmi2_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_bmi2_adjust_offset_nonzero @@ -2577,14 +2622,11 @@ MOVQ R14, R13 sequenceDecs_decodeSync_bmi2_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), CX - MOVQ 24(SP), R12 - LEAQ (CX)(R12*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), CX + MOVQ 56(SP), R12 MOVQ ctx+16(FP), R14 SUBQ R12, 104(R14) JS error_not_enough_literals @@ -2596,50 +2638,49 @@ JNZ sequenceDecs_decodeSync_bmi2_error_match_len_ofs_mismatch sequenceDecs_decodeSync_bmi2_match_len_ofs_ok: - MOVQ 24(SP), CX - MOVQ 8(SP), R12 - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ (CX)(R13*1), R14 + LEAQ 16(R12)(CX*1), R14 ADDQ R9, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ CX, CX - JZ check_offset - XORQ R14, R14 + MOVUPS (R10), X0 + MOVUPS X0, (R9) + CMPQ R12, $0x10 + JBE copy_1_end + MOVQ $0x00000010, R14 copy_1: MOVUPS (R10)(R14*1), X0 MOVUPS X0, (R9)(R14*1) ADDQ $0x10, R14 - CMPQ R14, CX + CMPQ R14, R12 JB copy_1 - ADDQ CX, R10 - ADDQ CX, R9 - ADDQ CX, R11 + +copy_1_end: + ADDQ R12, R10 + ADDQ R12, R9 + ADDQ R12, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) -check_offset: - MOVQ R11, CX - ADDQ 40(SP), CX - CMPQ R12, CX + MOVQ R11, R12 + ADDQ 72(SP), R12 + CMPQ R13, R12 JG error_match_off_too_big - CMPQ R12, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ R12, CX - SUBQ R11, CX + MOVQ R13, R12 + SUBQ R11, R12 JLS copy_match - MOVQ 48(SP), R14 - SUBQ CX, R14 - CMPQ R13, CX + MOVQ 80(SP), R14 + SUBQ R12, R14 + CMPQ CX, R12 JG copy_all_from_history - MOVQ R13, CX - SUBQ $0x10, CX + MOVQ CX, R12 + SUBQ $0x10, R12 JB copy_4_small copy_4_loop: @@ -2647,54 +2688,54 @@ MOVUPS X0, (R9) ADDQ $0x10, R14 ADDQ $0x10, R9 - SUBQ $0x10, CX + SUBQ $0x10, R12 JAE copy_4_loop - LEAQ 16(R14)(CX*1), R14 - LEAQ 16(R9)(CX*1), R9 + LEAQ 16(R14)(R12*1), R14 + LEAQ 16(R9)(R12*1), R9 MOVUPS -16(R14), X0 MOVUPS X0, -16(R9) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ CX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ CX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), CX - MOVB 2(R14), R12 - MOVW CX, (R9) - MOVB R12, 2(R9) - ADDQ R13, R14 - ADDQ R13, R9 - JMP copy_4_end + MOVWQZX (R14), R12 + MOVB 2(R14), R13 + MOVW R12, (R9) + MOVB R13, 2(R9) + ADDQ CX, R14 + ADDQ CX, R9 + JMP copy_4_end copy_4_move_4through7: - MOVL (R14), CX - MOVL -4(R14)(R13*1), R12 - MOVL CX, (R9) - MOVL R12, -4(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVL (R14), R12 + MOVL -4(R14)(CX*1), R13 + MOVL R12, (R9) + MOVL R13, -4(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), CX - MOVQ -8(R14)(R13*1), R12 - MOVQ CX, (R9) - MOVQ R12, -8(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVQ (R14), R12 + MOVQ -8(R14)(CX*1), R13 + MOVQ R12, (R9) + MOVQ R13, -8(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 copy_4_end: - ADDQ R13, R11 + ADDQ CX, R11 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ CX, R15 + MOVQ R12, R15 SUBQ $0x10, R15 JB copy_5_small @@ -2712,90 +2753,99 @@ JMP copy_5_end copy_5_small: - CMPQ CX, $0x03 + CMPQ R12, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ CX, $0x08 + CMPQ R12, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(CX*1), BP + MOVB -1(R14)(R12*1), BP MOVB R15, (R9) - MOVB BP, -1(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVB BP, -1(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R9) - MOVB BP, 2(R9) - ADDQ CX, R14 - ADDQ CX, R9 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R9) + MOVB BP, 2(R9) + ADDQ R12, R14 + ADDQ R12, R9 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(CX*1), BP + MOVL -4(R14)(R12*1), BP MOVL R15, (R9) - MOVL BP, -4(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVL BP, -4(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(CX*1), BP + MOVQ -8(R14)(R12*1), BP MOVQ R15, (R9) - MOVQ BP, -8(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVQ BP, -8(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 copy_5_end: - ADDQ CX, R11 - SUBQ CX, R13 + ADDQ R12, R11 + SUBQ R12, CX // Copy match from the current buffer copy_match: - MOVQ R9, CX - SUBQ R12, CX + MOVQ R9, R12 + SUBQ R13, R12 // ml <= mo - CMPQ R13, R12 + CMPQ CX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R11 - MOVQ R9, R12 - ADDQ R13, R9 + ADDQ CX, R11 + MOVUPS (R12), X0 + MOVUPS X0, (R9) + CMPQ CX, $0x10 + JA copy_2_long + ADDQ CX, R9 + JMP handle_loop + +copy_2_long: + LEAQ 16(R9), R13 + ADDQ CX, R9 + ADDQ $0x10, R12 + SUBQ $0x10, CX copy_2: - MOVUPS (CX), X0 - MOVUPS X0, (R12) - ADDQ $0x10, CX + MOVUPS (R12), X0 + MOVUPS X0, (R13) ADDQ $0x10, R12 - SUBQ $0x10, R13 + ADDQ $0x10, R13 + SUBQ $0x10, CX JHI copy_2 JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R11 + ADDQ CX, R11 copy_slow_3: - MOVB (CX), R12 - MOVB R12, (R9) - INCQ CX - INCQ R9 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (R12), R13 + MOVB R13, (R9) + INCQ R12 + INCQ R9 + DECQ CX + JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_bmi2_main_loop loop_finished: @@ -2804,6 +2854,13 @@ MOVB DL, 40(CX) MOVQ BX, 32(CX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R11, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R11, 136(AX) @@ -2817,7 +2874,7 @@ // Return with match length error sequenceDecs_decodeSync_bmi2_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -2826,7 +2883,7 @@ // Return with match too long error sequenceDecs_decodeSync_bmi2_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -2834,7 +2891,7 @@ // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R11, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -2843,7 +2900,7 @@ // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -2856,9 +2913,9 @@ // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R11, 136(AX) MOVQ $0x00000005, ret+24(FP) @@ -2866,7 +2923,7 @@ // func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: CMOV, SSE -TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $96-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX MOVBQZX 40(CX), BX @@ -2878,26 +2935,34 @@ MOVQ 72(AX), DI MOVQ 80(AX), R8 MOVQ 88(AX), R9 - XORQ CX, CX + MOVQ (AX), CX MOVQ CX, 8(SP) + MOVQ 24(AX), CX MOVQ CX, 16(SP) + MOVQ 48(AX), CX MOVQ CX, 24(SP) + MOVQ 96(AX), CX + MOVQ CX, 32(SP) + XORQ CX, CX + MOVQ CX, 40(SP) + MOVQ CX, 48(SP) + MOVQ CX, 56(SP) MOVQ 112(AX), R10 MOVQ 128(AX), CX - MOVQ CX, 32(SP) + MOVQ CX, 64(SP) MOVQ 144(AX), R11 MOVQ 136(AX), R12 MOVQ 200(AX), CX - MOVQ CX, 56(SP) + MOVQ CX, 88(SP) MOVQ 176(AX), CX - MOVQ CX, 48(SP) + MOVQ CX, 80(SP) MOVQ 184(AX), AX - MOVQ AX, 40(SP) - MOVQ 40(SP), AX - ADDQ AX, 48(SP) + MOVQ AX, 72(SP) + MOVQ 72(SP), AX + ADDQ AX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R10, 32(SP) + ADDQ R10, 64(SP) // outBase += outPosition ADDQ R12, R10 @@ -2935,46 +3000,32 @@ sequenceDecs_decodeSync_safe_amd64_fill_end: // Update offset - MOVQ R9, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_of_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_of_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_of_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_of_update_zero: - MOVQ AX, 8(SP) + MOVQ R9, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 40(SP) // Update match length - MOVQ R8, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_ml_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_ml_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_ml_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_ml_update_zero: - MOVQ AX, 16(SP) + MOVQ R8, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ SI, $0x08 @@ -3006,96 +3057,82 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_end: // Update literal length - MOVQ DI, AX - MOVQ BX, CX - MOVQ DX, R14 - SHLQ CL, R14 - MOVB AH, CL - SHRQ $0x20, AX - TESTQ CX, CX - JZ sequenceDecs_decodeSync_safe_amd64_ll_update_zero - ADDQ CX, BX - CMPQ BX, $0x40 - JA sequenceDecs_decodeSync_safe_amd64_ll_update_zero - CMPQ CX, $0x40 - JAE sequenceDecs_decodeSync_safe_amd64_ll_update_zero - NEGQ CX - SHRQ CL, R14 - ADDQ R14, AX - -sequenceDecs_decodeSync_safe_amd64_ll_update_zero: - MOVQ AX, 24(SP) + MOVQ DI, AX + MOVQ BX, CX + MOVQ DX, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + MOVBLZX AH, CX + SHRQ $0x20, AX + ADDQ CX, BX + XORQ $0x3f, CX + SHRQ CL, R14 + ADDQ R14, AX + MOVQ AX, 56(SP) // Fill bitreader for state updates MOVQ R13, (SP) MOVQ R9, AX SHRQ $0x08, AX MOVBQZX AL, AX - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_safe_amd64_skip_update // Update Literal Length State MOVBQZX DI, R13 SHRL $0x10, DI - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, DI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(DI*8), DI // Update Match Length State MOVBQZX R8, R13 SHRL $0x10, R8 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R8 // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(R8*8), R8 // Update Offset State MOVBQZX R9, R13 SHRL $0x10, R9 - LEAQ (BX)(R13*1), CX + MOVQ BX, CX MOVQ DX, R14 - MOVQ CX, BX - ROLQ CL, R14 - MOVL $0x00000001, R15 - MOVB R13, CL - SHLL CL, R15 - DECL R15 - ANDQ R15, R14 + SHLQ CL, R14 + SHRQ $0x01, R14 + ADDQ R13, BX + XORQ $0x3f, R13 + MOVQ R13, CX + SHRQ CL, R14 ADDQ R14, R9 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R9*8), R9 sequenceDecs_decodeSync_safe_amd64_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ AX, $0x01 JBE sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -3104,7 +3141,7 @@ JMP sequenceDecs_decodeSync_safe_amd64_after_adjust sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_safe_amd64_adjust_offset_nonzero @@ -3139,14 +3176,11 @@ MOVQ R14, R13 sequenceDecs_decodeSync_safe_amd64_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), AX - MOVQ 24(SP), CX - LEAQ (AX)(CX*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), AX + MOVQ 56(SP), CX MOVQ ctx+16(FP), R14 SUBQ CX, 104(R14) JS error_not_enough_literals @@ -3158,20 +3192,16 @@ JNZ sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok: - MOVQ 24(SP), AX - MOVQ 8(SP), CX - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ (AX)(R13*1), R14 + LEAQ (CX)(AX*1), R14 ADDQ R10, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ AX, AX + TESTQ CX, CX JZ check_offset - MOVQ AX, R14 + MOVQ CX, R14 SUBQ $0x10, R14 JB copy_1_small @@ -3189,70 +3219,70 @@ JMP copy_1_end copy_1_small: - CMPQ AX, $0x03 + CMPQ CX, $0x03 JE copy_1_move_3 JB copy_1_move_1or2 - CMPQ AX, $0x08 + CMPQ CX, $0x08 JB copy_1_move_4through7 JMP copy_1_move_8through16 copy_1_move_1or2: MOVB (R11), R14 - MOVB -1(R11)(AX*1), R15 + MOVB -1(R11)(CX*1), R15 MOVB R14, (R10) - MOVB R15, -1(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 + MOVB R15, -1(R10)(CX*1) + ADDQ CX, R11 + ADDQ CX, R10 JMP copy_1_end copy_1_move_3: - MOVW (R11), R14 - MOVB 2(R11), R15 - MOVW R14, (R10) - MOVB R15, 2(R10) - ADDQ AX, R11 - ADDQ AX, R10 - JMP copy_1_end + MOVWQZX (R11), R14 + MOVB 2(R11), R15 + MOVW R14, (R10) + MOVB R15, 2(R10) + ADDQ CX, R11 + ADDQ CX, R10 + JMP copy_1_end copy_1_move_4through7: MOVL (R11), R14 - MOVL -4(R11)(AX*1), R15 + MOVL -4(R11)(CX*1), R15 MOVL R14, (R10) - MOVL R15, -4(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 + MOVL R15, -4(R10)(CX*1) + ADDQ CX, R11 + ADDQ CX, R10 JMP copy_1_end copy_1_move_8through16: MOVQ (R11), R14 - MOVQ -8(R11)(AX*1), R15 + MOVQ -8(R11)(CX*1), R15 MOVQ R14, (R10) - MOVQ R15, -8(R10)(AX*1) - ADDQ AX, R11 - ADDQ AX, R10 + MOVQ R15, -8(R10)(CX*1) + ADDQ CX, R11 + ADDQ CX, R10 copy_1_end: - ADDQ AX, R12 + ADDQ CX, R12 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: - MOVQ R12, AX - ADDQ 40(SP), AX - CMPQ CX, AX + MOVQ R12, CX + ADDQ 72(SP), CX + CMPQ R13, CX JG error_match_off_too_big - CMPQ CX, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ CX, AX - SUBQ R12, AX + MOVQ R13, CX + SUBQ R12, CX JLS copy_match - MOVQ 48(SP), R14 - SUBQ AX, R14 - CMPQ R13, AX + MOVQ 80(SP), R14 + SUBQ CX, R14 + CMPQ AX, CX JG copy_all_from_history - MOVQ R13, AX - SUBQ $0x10, AX + MOVQ AX, CX + SUBQ $0x10, CX JB copy_4_small copy_4_loop: @@ -3260,54 +3290,54 @@ MOVUPS X0, (R10) ADDQ $0x10, R14 ADDQ $0x10, R10 - SUBQ $0x10, AX + SUBQ $0x10, CX JAE copy_4_loop - LEAQ 16(R14)(AX*1), R14 - LEAQ 16(R10)(AX*1), R10 + LEAQ 16(R14)(CX*1), R14 + LEAQ 16(R10)(CX*1), R10 MOVUPS -16(R14), X0 MOVUPS X0, -16(R10) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ AX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ AX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), AX - MOVB 2(R14), CL - MOVW AX, (R10) - MOVB CL, 2(R10) - ADDQ R13, R14 - ADDQ R13, R10 - JMP copy_4_end + MOVWQZX (R14), CX + MOVB 2(R14), R13 + MOVW CX, (R10) + MOVB R13, 2(R10) + ADDQ AX, R14 + ADDQ AX, R10 + JMP copy_4_end copy_4_move_4through7: - MOVL (R14), AX - MOVL -4(R14)(R13*1), CX - MOVL AX, (R10) - MOVL CX, -4(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVL (R14), CX + MOVL -4(R14)(AX*1), R13 + MOVL CX, (R10) + MOVL R13, -4(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), AX - MOVQ -8(R14)(R13*1), CX - MOVQ AX, (R10) - MOVQ CX, -8(R10)(R13*1) - ADDQ R13, R14 - ADDQ R13, R10 + MOVQ (R14), CX + MOVQ -8(R14)(AX*1), R13 + MOVQ CX, (R10) + MOVQ R13, -8(R10)(AX*1) + ADDQ AX, R14 + ADDQ AX, R10 copy_4_end: - ADDQ R13, R12 + ADDQ AX, R12 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ AX, R15 + MOVQ CX, R15 SUBQ $0x10, R15 JB copy_5_small @@ -3325,141 +3355,140 @@ JMP copy_5_end copy_5_small: - CMPQ AX, $0x03 + CMPQ CX, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ AX, $0x08 + CMPQ CX, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(AX*1), BP + MOVB -1(R14)(CX*1), BP MOVB R15, (R10) - MOVB BP, -1(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVB BP, -1(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R10) - MOVB BP, 2(R10) - ADDQ AX, R14 - ADDQ AX, R10 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R10) + MOVB BP, 2(R10) + ADDQ CX, R14 + ADDQ CX, R10 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(AX*1), BP + MOVL -4(R14)(CX*1), BP MOVL R15, (R10) - MOVL BP, -4(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVL BP, -4(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(AX*1), BP + MOVQ -8(R14)(CX*1), BP MOVQ R15, (R10) - MOVQ BP, -8(R10)(AX*1) - ADDQ AX, R14 - ADDQ AX, R10 + MOVQ BP, -8(R10)(CX*1) + ADDQ CX, R14 + ADDQ CX, R10 copy_5_end: - ADDQ AX, R12 - SUBQ AX, R13 + ADDQ CX, R12 + SUBQ CX, AX // Copy match from the current buffer copy_match: - MOVQ R10, AX - SUBQ CX, AX + MOVQ R10, CX + SUBQ R13, CX // ml <= mo - CMPQ R13, CX + CMPQ AX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R12 - MOVQ R13, CX - SUBQ $0x10, CX + ADDQ AX, R12 + MOVQ AX, R13 + SUBQ $0x10, R13 JB copy_2_small copy_2_loop: - MOVUPS (AX), X0 + MOVUPS (CX), X0 MOVUPS X0, (R10) - ADDQ $0x10, AX + ADDQ $0x10, CX ADDQ $0x10, R10 - SUBQ $0x10, CX + SUBQ $0x10, R13 JAE copy_2_loop - LEAQ 16(AX)(CX*1), AX - LEAQ 16(R10)(CX*1), R10 - MOVUPS -16(AX), X0 + LEAQ 16(CX)(R13*1), CX + LEAQ 16(R10)(R13*1), R10 + MOVUPS -16(CX), X0 MOVUPS X0, -16(R10) JMP copy_2_end copy_2_small: - CMPQ R13, $0x03 + CMPQ AX, $0x03 JE copy_2_move_3 JB copy_2_move_1or2 - CMPQ R13, $0x08 + CMPQ AX, $0x08 JB copy_2_move_4through7 JMP copy_2_move_8through16 copy_2_move_1or2: - MOVB (AX), CL - MOVB -1(AX)(R13*1), R14 - MOVB CL, (R10) - MOVB R14, -1(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 + MOVB (CX), R13 + MOVB -1(CX)(AX*1), R14 + MOVB R13, (R10) + MOVB R14, -1(R10)(AX*1) + ADDQ AX, CX + ADDQ AX, R10 JMP copy_2_end copy_2_move_3: - MOVW (AX), CX - MOVB 2(AX), R14 - MOVW CX, (R10) - MOVB R14, 2(R10) - ADDQ R13, AX - ADDQ R13, R10 - JMP copy_2_end + MOVWQZX (CX), R13 + MOVB 2(CX), R14 + MOVW R13, (R10) + MOVB R14, 2(R10) + ADDQ AX, CX + ADDQ AX, R10 + JMP copy_2_end copy_2_move_4through7: - MOVL (AX), CX - MOVL -4(AX)(R13*1), R14 - MOVL CX, (R10) - MOVL R14, -4(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 + MOVL (CX), R13 + MOVL -4(CX)(AX*1), R14 + MOVL R13, (R10) + MOVL R14, -4(R10)(AX*1) + ADDQ AX, CX + ADDQ AX, R10 JMP copy_2_end copy_2_move_8through16: - MOVQ (AX), CX - MOVQ -8(AX)(R13*1), R14 - MOVQ CX, (R10) - MOVQ R14, -8(R10)(R13*1) - ADDQ R13, AX - ADDQ R13, R10 + MOVQ (CX), R13 + MOVQ -8(CX)(AX*1), R14 + MOVQ R13, (R10) + MOVQ R14, -8(R10)(AX*1) + ADDQ AX, CX + ADDQ AX, R10 copy_2_end: JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R12 + ADDQ AX, R12 copy_slow_3: - MOVB (AX), CL - MOVB CL, (R10) - INCQ AX - INCQ R10 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (CX), R13 + MOVB R13, (R10) + INCQ CX + INCQ R10 + DECQ AX + JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), AX - DECQ 96(AX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_safe_amd64_main_loop loop_finished: @@ -3468,6 +3497,13 @@ MOVB BL, 40(AX) MOVQ SI, 32(AX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R12, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R12, 136(AX) @@ -3481,7 +3517,7 @@ // Return with match length error sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -3490,7 +3526,7 @@ // Return with match too long error sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -3498,7 +3534,7 @@ // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R12, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -3507,7 +3543,7 @@ // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -3520,9 +3556,9 @@ // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R12, 136(AX) MOVQ $0x00000005, ret+24(FP) @@ -3530,7 +3566,7 @@ // func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // Requires: BMI, BMI2, CMOV, SSE -TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32 +TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $96-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX MOVBQZX 40(BX), DX @@ -3542,26 +3578,34 @@ MOVQ 72(CX), SI MOVQ 80(CX), DI MOVQ 88(CX), R8 - XORQ R9, R9 + MOVQ (CX), R9 MOVQ R9, 8(SP) + MOVQ 24(CX), R9 MOVQ R9, 16(SP) + MOVQ 48(CX), R9 MOVQ R9, 24(SP) + MOVQ 96(CX), R9 + MOVQ R9, 32(SP) + XORQ R9, R9 + MOVQ R9, 40(SP) + MOVQ R9, 48(SP) + MOVQ R9, 56(SP) MOVQ 112(CX), R9 MOVQ 128(CX), R10 - MOVQ R10, 32(SP) + MOVQ R10, 64(SP) MOVQ 144(CX), R10 MOVQ 136(CX), R11 MOVQ 200(CX), R12 - MOVQ R12, 56(SP) + MOVQ R12, 88(SP) MOVQ 176(CX), R12 - MOVQ R12, 48(SP) + MOVQ R12, 80(SP) MOVQ 184(CX), CX - MOVQ CX, 40(SP) - MOVQ 40(SP), CX - ADDQ CX, 48(SP) + MOVQ CX, 72(SP) + MOVQ 72(SP), CX + ADDQ CX, 80(SP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - ADDQ R9, 32(SP) + ADDQ R9, 64(SP) // outBase += outPosition ADDQ R11, R9 @@ -3609,7 +3653,7 @@ MOVQ R8, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 8(SP) + MOVQ CX, 40(SP) // Update match length MOVQ $0x00000808, CX @@ -3622,7 +3666,7 @@ MOVQ DI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 16(SP) + MOVQ CX, 48(SP) // Fill bitreader to have enough for the remaining CMPQ BX, $0x08 @@ -3664,14 +3708,13 @@ MOVQ SI, CX SHRQ $0x20, CX ADDQ R14, CX - MOVQ CX, 24(SP) + MOVQ CX, 56(SP) // Fill bitreader for state updates MOVQ R12, (SP) MOVQ $0x00000808, CX BEXTRQ CX, R8, R12 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 + CMPQ 32(SP), $0x00 JZ sequenceDecs_decodeSync_safe_bmi2_skip_update LEAQ (SI)(DI*1), R13 ADDQ R8, R13 @@ -3689,8 +3732,7 @@ ADDQ CX, R8 // Load ctx.ofTable - MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX + MOVQ 24(SP), CX MOVQ (CX)(R8*8), R8 // Update Match Length State @@ -3700,8 +3742,7 @@ ADDQ CX, DI // Load ctx.mlTable - MOVQ ctx+16(FP), CX - MOVQ 24(CX), CX + MOVQ 16(SP), CX MOVQ (CX)(DI*8), DI // Update Literal Length State @@ -3710,14 +3751,13 @@ ADDQ CX, SI // Load ctx.llTable - MOVQ ctx+16(FP), CX - MOVQ (CX), CX + MOVQ 8(SP), CX MOVQ (CX)(SI*8), SI sequenceDecs_decodeSync_safe_bmi2_skip_update: // Adjust offset MOVQ s+0(FP), CX - MOVQ 8(SP), R13 + MOVQ 40(SP), R13 CMPQ R12, $0x01 JBE sequenceDecs_decodeSync_safe_bmi2_adjust_offsetB_1_or_0 MOVUPS 144(CX), X0 @@ -3726,7 +3766,7 @@ JMP sequenceDecs_decodeSync_safe_bmi2_after_adjust sequenceDecs_decodeSync_safe_bmi2_adjust_offsetB_1_or_0: - CMPQ 24(SP), $0x00000000 + CMPQ 56(SP), $0x00000000 JNE sequenceDecs_decodeSync_safe_bmi2_adjust_offset_maybezero INCQ R13 JMP sequenceDecs_decodeSync_safe_bmi2_adjust_offset_nonzero @@ -3761,14 +3801,11 @@ MOVQ R14, R13 sequenceDecs_decodeSync_safe_bmi2_after_adjust: - MOVQ R13, 8(SP) + MOVQ R13, 40(SP) // Check values - MOVQ 16(SP), CX - MOVQ 24(SP), R12 - LEAQ (CX)(R12*1), R14 - MOVQ s+0(FP), R15 - ADDQ R14, 256(R15) + MOVQ 48(SP), CX + MOVQ 56(SP), R12 MOVQ ctx+16(FP), R14 SUBQ R12, 104(R14) JS error_not_enough_literals @@ -3780,20 +3817,16 @@ JNZ sequenceDecs_decodeSync_safe_bmi2_error_match_len_ofs_mismatch sequenceDecs_decodeSync_safe_bmi2_match_len_ofs_ok: - MOVQ 24(SP), CX - MOVQ 8(SP), R12 - MOVQ 16(SP), R13 - // Check if we have enough space in s.out - LEAQ (CX)(R13*1), R14 + LEAQ (R12)(CX*1), R14 ADDQ R9, R14 - CMPQ R14, 32(SP) + CMPQ R14, 64(SP) JA error_not_enough_space // Copy literals - TESTQ CX, CX + TESTQ R12, R12 JZ check_offset - MOVQ CX, R14 + MOVQ R12, R14 SUBQ $0x10, R14 JB copy_1_small @@ -3811,70 +3844,70 @@ JMP copy_1_end copy_1_small: - CMPQ CX, $0x03 + CMPQ R12, $0x03 JE copy_1_move_3 JB copy_1_move_1or2 - CMPQ CX, $0x08 + CMPQ R12, $0x08 JB copy_1_move_4through7 JMP copy_1_move_8through16 copy_1_move_1or2: MOVB (R10), R14 - MOVB -1(R10)(CX*1), R15 + MOVB -1(R10)(R12*1), R15 MOVB R14, (R9) - MOVB R15, -1(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 + MOVB R15, -1(R9)(R12*1) + ADDQ R12, R10 + ADDQ R12, R9 JMP copy_1_end copy_1_move_3: - MOVW (R10), R14 - MOVB 2(R10), R15 - MOVW R14, (R9) - MOVB R15, 2(R9) - ADDQ CX, R10 - ADDQ CX, R9 - JMP copy_1_end + MOVWQZX (R10), R14 + MOVB 2(R10), R15 + MOVW R14, (R9) + MOVB R15, 2(R9) + ADDQ R12, R10 + ADDQ R12, R9 + JMP copy_1_end copy_1_move_4through7: MOVL (R10), R14 - MOVL -4(R10)(CX*1), R15 + MOVL -4(R10)(R12*1), R15 MOVL R14, (R9) - MOVL R15, -4(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 + MOVL R15, -4(R9)(R12*1) + ADDQ R12, R10 + ADDQ R12, R9 JMP copy_1_end copy_1_move_8through16: MOVQ (R10), R14 - MOVQ -8(R10)(CX*1), R15 + MOVQ -8(R10)(R12*1), R15 MOVQ R14, (R9) - MOVQ R15, -8(R9)(CX*1) - ADDQ CX, R10 - ADDQ CX, R9 + MOVQ R15, -8(R9)(R12*1) + ADDQ R12, R10 + ADDQ R12, R9 copy_1_end: - ADDQ CX, R11 + ADDQ R12, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: - MOVQ R11, CX - ADDQ 40(SP), CX - CMPQ R12, CX + MOVQ R11, R12 + ADDQ 72(SP), R12 + CMPQ R13, R12 JG error_match_off_too_big - CMPQ R12, 56(SP) + CMPQ R13, 88(SP) JG error_match_off_too_big // Copy match from history - MOVQ R12, CX - SUBQ R11, CX + MOVQ R13, R12 + SUBQ R11, R12 JLS copy_match - MOVQ 48(SP), R14 - SUBQ CX, R14 - CMPQ R13, CX + MOVQ 80(SP), R14 + SUBQ R12, R14 + CMPQ CX, R12 JG copy_all_from_history - MOVQ R13, CX - SUBQ $0x10, CX + MOVQ CX, R12 + SUBQ $0x10, R12 JB copy_4_small copy_4_loop: @@ -3882,54 +3915,54 @@ MOVUPS X0, (R9) ADDQ $0x10, R14 ADDQ $0x10, R9 - SUBQ $0x10, CX + SUBQ $0x10, R12 JAE copy_4_loop - LEAQ 16(R14)(CX*1), R14 - LEAQ 16(R9)(CX*1), R9 + LEAQ 16(R14)(R12*1), R14 + LEAQ 16(R9)(R12*1), R9 MOVUPS -16(R14), X0 MOVUPS X0, -16(R9) JMP copy_4_end copy_4_small: - CMPQ R13, $0x03 + CMPQ CX, $0x03 JE copy_4_move_3 - CMPQ R13, $0x08 + CMPQ CX, $0x08 JB copy_4_move_4through7 JMP copy_4_move_8through16 copy_4_move_3: - MOVW (R14), CX - MOVB 2(R14), R12 - MOVW CX, (R9) - MOVB R12, 2(R9) - ADDQ R13, R14 - ADDQ R13, R9 - JMP copy_4_end + MOVWQZX (R14), R12 + MOVB 2(R14), R13 + MOVW R12, (R9) + MOVB R13, 2(R9) + ADDQ CX, R14 + ADDQ CX, R9 + JMP copy_4_end copy_4_move_4through7: - MOVL (R14), CX - MOVL -4(R14)(R13*1), R12 - MOVL CX, (R9) - MOVL R12, -4(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVL (R14), R12 + MOVL -4(R14)(CX*1), R13 + MOVL R12, (R9) + MOVL R13, -4(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 JMP copy_4_end copy_4_move_8through16: - MOVQ (R14), CX - MOVQ -8(R14)(R13*1), R12 - MOVQ CX, (R9) - MOVQ R12, -8(R9)(R13*1) - ADDQ R13, R14 - ADDQ R13, R9 + MOVQ (R14), R12 + MOVQ -8(R14)(CX*1), R13 + MOVQ R12, (R9) + MOVQ R13, -8(R9)(CX*1) + ADDQ CX, R14 + ADDQ CX, R9 copy_4_end: - ADDQ R13, R11 + ADDQ CX, R11 JMP handle_loop JMP loop_finished copy_all_from_history: - MOVQ CX, R15 + MOVQ R12, R15 SUBQ $0x10, R15 JB copy_5_small @@ -3947,141 +3980,140 @@ JMP copy_5_end copy_5_small: - CMPQ CX, $0x03 + CMPQ R12, $0x03 JE copy_5_move_3 JB copy_5_move_1or2 - CMPQ CX, $0x08 + CMPQ R12, $0x08 JB copy_5_move_4through7 JMP copy_5_move_8through16 copy_5_move_1or2: MOVB (R14), R15 - MOVB -1(R14)(CX*1), BP + MOVB -1(R14)(R12*1), BP MOVB R15, (R9) - MOVB BP, -1(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVB BP, -1(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_3: - MOVW (R14), R15 - MOVB 2(R14), BP - MOVW R15, (R9) - MOVB BP, 2(R9) - ADDQ CX, R14 - ADDQ CX, R9 - JMP copy_5_end + MOVWQZX (R14), R15 + MOVB 2(R14), BP + MOVW R15, (R9) + MOVB BP, 2(R9) + ADDQ R12, R14 + ADDQ R12, R9 + JMP copy_5_end copy_5_move_4through7: MOVL (R14), R15 - MOVL -4(R14)(CX*1), BP + MOVL -4(R14)(R12*1), BP MOVL R15, (R9) - MOVL BP, -4(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVL BP, -4(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 JMP copy_5_end copy_5_move_8through16: MOVQ (R14), R15 - MOVQ -8(R14)(CX*1), BP + MOVQ -8(R14)(R12*1), BP MOVQ R15, (R9) - MOVQ BP, -8(R9)(CX*1) - ADDQ CX, R14 - ADDQ CX, R9 + MOVQ BP, -8(R9)(R12*1) + ADDQ R12, R14 + ADDQ R12, R9 copy_5_end: - ADDQ CX, R11 - SUBQ CX, R13 + ADDQ R12, R11 + SUBQ R12, CX // Copy match from the current buffer copy_match: - MOVQ R9, CX - SUBQ R12, CX + MOVQ R9, R12 + SUBQ R13, R12 // ml <= mo - CMPQ R13, R12 + CMPQ CX, R13 JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, R11 - MOVQ R13, R12 - SUBQ $0x10, R12 + ADDQ CX, R11 + MOVQ CX, R13 + SUBQ $0x10, R13 JB copy_2_small copy_2_loop: - MOVUPS (CX), X0 + MOVUPS (R12), X0 MOVUPS X0, (R9) - ADDQ $0x10, CX + ADDQ $0x10, R12 ADDQ $0x10, R9 - SUBQ $0x10, R12 + SUBQ $0x10, R13 JAE copy_2_loop - LEAQ 16(CX)(R12*1), CX - LEAQ 16(R9)(R12*1), R9 - MOVUPS -16(CX), X0 + LEAQ 16(R12)(R13*1), R12 + LEAQ 16(R9)(R13*1), R9 + MOVUPS -16(R12), X0 MOVUPS X0, -16(R9) JMP copy_2_end copy_2_small: - CMPQ R13, $0x03 + CMPQ CX, $0x03 JE copy_2_move_3 JB copy_2_move_1or2 - CMPQ R13, $0x08 + CMPQ CX, $0x08 JB copy_2_move_4through7 JMP copy_2_move_8through16 copy_2_move_1or2: - MOVB (CX), R12 - MOVB -1(CX)(R13*1), R14 - MOVB R12, (R9) - MOVB R14, -1(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 + MOVB (R12), R13 + MOVB -1(R12)(CX*1), R14 + MOVB R13, (R9) + MOVB R14, -1(R9)(CX*1) + ADDQ CX, R12 + ADDQ CX, R9 JMP copy_2_end copy_2_move_3: - MOVW (CX), R12 - MOVB 2(CX), R14 - MOVW R12, (R9) - MOVB R14, 2(R9) - ADDQ R13, CX - ADDQ R13, R9 - JMP copy_2_end + MOVWQZX (R12), R13 + MOVB 2(R12), R14 + MOVW R13, (R9) + MOVB R14, 2(R9) + ADDQ CX, R12 + ADDQ CX, R9 + JMP copy_2_end copy_2_move_4through7: - MOVL (CX), R12 - MOVL -4(CX)(R13*1), R14 - MOVL R12, (R9) - MOVL R14, -4(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 + MOVL (R12), R13 + MOVL -4(R12)(CX*1), R14 + MOVL R13, (R9) + MOVL R14, -4(R9)(CX*1) + ADDQ CX, R12 + ADDQ CX, R9 JMP copy_2_end copy_2_move_8through16: - MOVQ (CX), R12 - MOVQ -8(CX)(R13*1), R14 - MOVQ R12, (R9) - MOVQ R14, -8(R9)(R13*1) - ADDQ R13, CX - ADDQ R13, R9 + MOVQ (R12), R13 + MOVQ -8(R12)(CX*1), R14 + MOVQ R13, (R9) + MOVQ R14, -8(R9)(CX*1) + ADDQ CX, R12 + ADDQ CX, R9 copy_2_end: JMP handle_loop // Copy overlapping match copy_overlapping_match: - ADDQ R13, R11 + ADDQ CX, R11 copy_slow_3: - MOVB (CX), R12 - MOVB R12, (R9) - INCQ CX - INCQ R9 - DECQ R13 - JNZ copy_slow_3 + MOVBQZX (R12), R13 + MOVB R13, (R9) + INCQ R12 + INCQ R9 + DECQ CX + JNZ copy_slow_3 handle_loop: - MOVQ ctx+16(FP), CX - DECQ 96(CX) + DECQ 32(SP) JNS sequenceDecs_decodeSync_safe_bmi2_main_loop loop_finished: @@ -4090,6 +4122,13 @@ MOVB DL, 40(CX) MOVQ BX, 32(CX) + // s.seqSize += outPosition - ctx.outPosition + MOVQ ctx+16(FP), AX + MOVQ s+0(FP), CX + MOVQ R11, DX + SUBQ 136(AX), DX + ADDQ DX, 256(CX) + // Update the context MOVQ ctx+16(FP), AX MOVQ R11, 136(AX) @@ -4103,7 +4142,7 @@ // Return with match length error sequenceDecs_decodeSync_safe_bmi2_error_match_len_ofs_mismatch: - MOVQ 16(SP), AX + MOVQ 48(SP), AX MOVQ ctx+16(FP), CX MOVQ AX, 216(CX) MOVQ $0x00000001, ret+24(FP) @@ -4112,7 +4151,7 @@ // Return with match too long error sequenceDecs_decodeSync_safe_bmi2_error_match_len_too_big: MOVQ ctx+16(FP), AX - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ $0x00000002, ret+24(FP) RET @@ -4120,7 +4159,7 @@ // Return with match offset too long error error_match_off_too_big: MOVQ ctx+16(FP), AX - MOVQ 8(SP), CX + MOVQ 40(SP), CX MOVQ CX, 224(AX) MOVQ R11, 136(AX) MOVQ $0x00000003, ret+24(FP) @@ -4129,7 +4168,7 @@ // Return with not enough literals error error_not_enough_literals: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) MOVQ $0x00000004, ret+24(FP) RET @@ -4142,9 +4181,9 @@ // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX - MOVQ 24(SP), CX + MOVQ 56(SP), CX MOVQ CX, 208(AX) - MOVQ 16(SP), CX + MOVQ 48(SP), CX MOVQ CX, 216(AX) MOVQ R11, 136(AX) MOVQ $0x00000005, ret+24(FP) --- a/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || appengine || !gc || noasm package zstd --- a/vendor/github.com/klauspost/compress/zstd/snappy.go +++ b/vendor/github.com/klauspost/compress/zstd/snappy.go @@ -69,6 +69,10 @@ // No CRC value is being generated and not all CRC values of the Snappy stream are checked. // However, it provides really fast recompression of Snappy streams. // The converter can be reused to avoid allocations, even after errors. +// +// Deprecated: Decode the stream with s2.NewReader, which reads Snappy streams, +// and compress the output with an Encoder. This validates the input and +// compresses much better. type SnappyConverter struct { r io.Reader err error @@ -334,9 +338,10 @@ return errUnsupportedLiteralLength } - //if length > snappyMaxBlockSize-d || uint32(length) > len(src)-s { - // return ErrSnappyCorrupt - //} + if length > len(src)-s { + println("length > len(src)-s", length, len(src)-s) + return ErrSnappyCorrupt + } blk.literals = append(blk.literals, src[s:s+length]...) //println(length, "litLen") @@ -372,6 +377,14 @@ offset = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 } + // A zstd match is at least zstdMinMatch bytes. A shorter copy + // cannot be represented as a sequence and would underflow the + // matchLen computation below, producing an out-of-range match + // length code. + if length < zstdMinMatch { + return ErrSnappyCorrupt + } + if offset <= 0 || blk.size+lits < int(offset) /*|| length > len(blk)-d */ { println("offset <= 0 || blk.size+lits < int(offset)", offset, blk.size+lits, int(offset), blk.size, lits) --- a/vendor/golang.org/x/net/internal/socket/socket.go +++ b/vendor/golang.org/x/net/internal/socket/socket.go @@ -129,7 +129,8 @@ // MarshalHeader marshals the header fields of the control message at // the head on m. func (m ControlMessage) MarshalHeader(lvl, typ, dataLen int) error { - if len(m) < controlHeaderLen() { + l := controlHeaderLen() + if l <= 0 || len(m) < l { return errors.New("short message") } h := (*cmsghdr)(unsafe.Pointer(&m[0])) @@ -141,7 +142,7 @@ // message at the head on m. func (m ControlMessage) ParseHeader() (lvl, typ, dataLen int, err error) { l := controlHeaderLen() - if len(m) < l { + if l <= 0 || len(m) < l { return 0, 0, 0, errors.New("short message") } h := (*cmsghdr)(unsafe.Pointer(&m[0])) @@ -152,7 +153,7 @@ // the next control message. func (m ControlMessage) Marshal(lvl, typ int, data []byte) (ControlMessage, error) { l := len(data) - if len(m) < ControlMessageSpace(l) { + if controlHeaderLen() <= 0 || len(m) < ControlMessageSpace(l) { return nil, errors.New("short message") } h := (*cmsghdr)(unsafe.Pointer(&m[0])) @@ -167,6 +168,9 @@ // // Parse works for both standard and compatible messages. func (m ControlMessage) Parse() ([]ControlMessage, error) { + if controlHeaderLen() <= 0 { + return nil, nil + } var ms []ControlMessage for len(m) >= controlHeaderLen() { h := (*cmsghdr)(unsafe.Pointer(&m[0])) --- a/vendor/golang.org/x/net/ipv4/batch.go +++ b/vendor/golang.org/x/net/ipv4/batch.go @@ -75,6 +75,9 @@ if !c.ok() { return 0, errInvalidConn } + if len(ms) == 0 { + return 0, nil + } switch runtime.GOOS { case "linux": n, err := c.RecvMsgs([]socket.Message(ms), flags) @@ -109,6 +112,9 @@ if !c.ok() { return 0, errInvalidConn } + if len(ms) == 0 { + return 0, nil + } switch runtime.GOOS { case "linux": n, err := c.SendMsgs([]socket.Message(ms), flags) @@ -141,6 +147,9 @@ if !c.ok() { return 0, errInvalidConn } + if len(ms) == 0 { + return 0, nil + } switch runtime.GOOS { case "linux": n, err := c.RecvMsgs([]socket.Message(ms), flags) @@ -175,6 +184,9 @@ if !c.ok() { return 0, errInvalidConn } + if len(ms) == 0 { + return 0, nil + } switch runtime.GOOS { case "linux": n, err := c.SendMsgs([]socket.Message(ms), flags) --- a/vendor/golang.org/x/net/ipv4/control.go +++ b/vendor/golang.org/x/net/ipv4/control.go @@ -84,13 +84,13 @@ continue } switch { - case typ == ctlOpts[ctlTTL].name && l >= ctlOpts[ctlTTL].length: + case ctlOpts[ctlTTL].name > 0 && typ == ctlOpts[ctlTTL].name && l >= ctlOpts[ctlTTL].length: ctlOpts[ctlTTL].parse(cm, m.Data(l)) - case typ == ctlOpts[ctlDst].name && l >= ctlOpts[ctlDst].length: + case ctlOpts[ctlDst].name > 0 && typ == ctlOpts[ctlDst].name && l >= ctlOpts[ctlDst].length: ctlOpts[ctlDst].parse(cm, m.Data(l)) - case typ == ctlOpts[ctlInterface].name && l >= ctlOpts[ctlInterface].length: + case ctlOpts[ctlInterface].name > 0 && typ == ctlOpts[ctlInterface].name && l >= ctlOpts[ctlInterface].length: ctlOpts[ctlInterface].parse(cm, m.Data(l)) - case typ == ctlOpts[ctlPacketInfo].name && l >= ctlOpts[ctlPacketInfo].length: + case ctlOpts[ctlPacketInfo].name > 0 && typ == ctlOpts[ctlPacketInfo].name && l >= ctlOpts[ctlPacketInfo].length: ctlOpts[ctlPacketInfo].parse(cm, m.Data(l)) } } --- a/vendor/golang.org/x/net/ipv4/control_unix.go +++ b/vendor/golang.org/x/net/ipv4/control_unix.go @@ -7,7 +7,8 @@ package ipv4 import ( - "unsafe" + "encoding/binary" + "runtime" "golang.org/x/net/internal/iana" "golang.org/x/net/internal/socket" @@ -66,10 +67,18 @@ func marshalTTL(b []byte, cm *ControlMessage) []byte { m := socket.ControlMessage(b) + if runtime.GOOS == "linux" { + m.MarshalHeader(iana.ProtocolIP, unix.IP_TTL, 4) + return m.Next(4) + } m.MarshalHeader(iana.ProtocolIP, unix.IP_RECVTTL, 1) return m.Next(1) } func parseTTL(cm *ControlMessage, b []byte) { - cm.TTL = int(*(*byte)(unsafe.Pointer(&b[:1][0]))) + if runtime.GOOS == "linux" { + cm.TTL = int(binary.NativeEndian.Uint32(b[:4])) + return + } + cm.TTL = int(b[0]) } --- a/vendor/golang.org/x/net/ipv4/header.go +++ b/vendor/golang.org/x/net/ipv4/header.go @@ -111,6 +111,9 @@ return errHeaderTooShort } hdrlen := int(b[0]&0x0f) << 2 + if hdrlen < HeaderLen { + return errHeaderTooShort + } if len(b) < hdrlen { return errExtHeaderTooShort } --- a/vendor/golang.org/x/net/ipv4/packet.go +++ b/vendor/golang.org/x/net/ipv4/packet.go @@ -65,6 +65,12 @@ return nil, nil, errHeaderTooShort } hdrlen := int(b[0]&0x0f) << 2 + if hdrlen < HeaderLen { + return nil, nil, errHeaderTooShort + } + if len(b) < hdrlen { + return nil, nil, errExtHeaderTooShort + } return b[:hdrlen], b[hdrlen:], nil } --- a/vendor/golang.org/x/net/ipv4/payload_cmsg.go +++ b/vendor/golang.org/x/net/ipv4/payload_cmsg.go @@ -38,13 +38,17 @@ return 0, nil, nil, &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} } hdrlen := int(h[0]&0x0f) << 2 + if m.N < HeaderLen || hdrlen < HeaderLen { + return 0, nil, nil, &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: errHeaderTooShort} + } + if m.N < hdrlen { + return 0, nil, nil, &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: errExtHeaderTooShort} + } if hdrlen > len(h) { d := hdrlen - len(h) copy(b, b[d:]) - m.N -= d - } else { - m.N -= hdrlen } + m.N -= hdrlen default: return 0, nil, nil, &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: errInvalidConnType} } --- a/vendor/golang.org/x/net/ipv4/sys_bpf.go +++ b/vendor/golang.org/x/net/ipv4/sys_bpf.go @@ -7,6 +7,8 @@ package ipv4 import ( + "errors" + "math" "unsafe" "golang.org/x/net/bpf" @@ -15,6 +17,9 @@ ) func (so *sockOpt) setAttachFilter(c *socket.Conn, f []bpf.RawInstruction) error { + if len(f) == 0 || len(f) > math.MaxUint16 { + return errors.New("invalid filter") + } prog := unix.SockFprog{ Len: uint16(len(f)), Filter: (*unix.SockFilter)(unsafe.Pointer(&f[0])), --- a/vendor/golang.org/x/net/ipv4/sys_linux.go +++ b/vendor/golang.org/x/net/ipv4/sys_linux.go @@ -17,7 +17,7 @@ var ( ctlOpts = [ctlMax]ctlOpt{ - ctlTTL: {unix.IP_TTL, 1, marshalTTL, parseTTL}, + ctlTTL: {unix.IP_TTL, 4, marshalTTL, parseTTL}, ctlPacketInfo: {unix.IP_PKTINFO, sizeofInetPktinfo, marshalPacketInfo, parsePacketInfo}, } --- a/vendor/golang.org/x/net/ipv4/sys_solaris.go +++ b/vendor/golang.org/x/net/ipv4/sys_solaris.go @@ -19,7 +19,7 @@ var ( ctlOpts = [ctlMax]ctlOpt{ - ctlTTL: {unix.IP_RECVTTL, 4, marshalTTL, parseTTL}, + ctlTTL: {unix.IP_RECVTTL, 1, marshalTTL, parseTTL}, ctlPacketInfo: {unix.IP_PKTINFO, sizeofInetPktinfo, marshalPacketInfo, parsePacketInfo}, } --- a/vendor/golang.org/x/net/ipv6/batch.go +++ b/vendor/golang.org/x/net/ipv6/batch.go @@ -66,6 +66,9 @@ if !c.ok() { return 0, errInvalidConn } + if len(ms) == 0 { + return 0, nil + } switch runtime.GOOS { case "linux": n, err := c.RecvMsgs([]socket.Message(ms), flags) @@ -97,6 +100,9 @@ if !c.ok() { return 0, errInvalidConn } + if len(ms) == 0 { + return 0, nil + } switch runtime.GOOS { case "linux": n, err := c.SendMsgs([]socket.Message(ms), flags) --- a/vendor/golang.org/x/net/ipv6/control.go +++ b/vendor/golang.org/x/net/ipv6/control.go @@ -130,13 +130,13 @@ continue } switch { - case typ == ctlOpts[ctlTrafficClass].name && l >= ctlOpts[ctlTrafficClass].length: + case ctlOpts[ctlTrafficClass].name > 0 && typ == ctlOpts[ctlTrafficClass].name && l >= ctlOpts[ctlTrafficClass].length: ctlOpts[ctlTrafficClass].parse(cm, m.Data(l)) - case typ == ctlOpts[ctlHopLimit].name && l >= ctlOpts[ctlHopLimit].length: + case ctlOpts[ctlHopLimit].name > 0 && typ == ctlOpts[ctlHopLimit].name && l >= ctlOpts[ctlHopLimit].length: ctlOpts[ctlHopLimit].parse(cm, m.Data(l)) - case typ == ctlOpts[ctlPacketInfo].name && l >= ctlOpts[ctlPacketInfo].length: + case ctlOpts[ctlPacketInfo].name > 0 && typ == ctlOpts[ctlPacketInfo].name && l >= ctlOpts[ctlPacketInfo].length: ctlOpts[ctlPacketInfo].parse(cm, m.Data(l)) - case typ == ctlOpts[ctlPathMTU].name && l >= ctlOpts[ctlPathMTU].length: + case ctlOpts[ctlPathMTU].name > 0 && typ == ctlOpts[ctlPathMTU].name && l >= ctlOpts[ctlPathMTU].length: ctlOpts[ctlPathMTU].parse(cm, m.Data(l)) } } --- a/vendor/golang.org/x/net/ipv6/icmp_bsd.go +++ b/vendor/golang.org/x/net/ipv6/icmp_bsd.go @@ -7,10 +7,16 @@ package ipv6 func (f *icmpv6Filter) accept(typ ICMPType) { + if typ < 0 || int(typ>>5) >= len(f.Filt) { + return + } f.Filt[typ>>5] |= 1 << (uint32(typ) & 31) } func (f *icmpv6Filter) block(typ ICMPType) { + if typ < 0 || int(typ>>5) >= len(f.Filt) { + return + } f.Filt[typ>>5] &^= 1 << (uint32(typ) & 31) } @@ -25,5 +31,8 @@ } func (f *icmpv6Filter) willBlock(typ ICMPType) bool { + if typ < 0 || int(typ>>5) >= len(f.Filt) { + return false + } return f.Filt[typ>>5]&(1<<(uint32(typ)&31)) == 0 } --- a/vendor/golang.org/x/net/ipv6/icmp_linux.go +++ b/vendor/golang.org/x/net/ipv6/icmp_linux.go @@ -5,10 +5,16 @@ package ipv6 func (f *icmpv6Filter) accept(typ ICMPType) { + if typ < 0 || int(typ>>5) >= len(f.Data) { + return + } f.Data[typ>>5] &^= 1 << (uint32(typ) & 31) } func (f *icmpv6Filter) block(typ ICMPType) { + if typ < 0 || int(typ>>5) >= len(f.Data) { + return + } f.Data[typ>>5] |= 1 << (uint32(typ) & 31) } @@ -23,5 +29,8 @@ } func (f *icmpv6Filter) willBlock(typ ICMPType) bool { + if typ < 0 || int(typ>>5) >= len(f.Data) { + return false + } return f.Data[typ>>5]&(1<<(uint32(typ)&31)) != 0 } --- a/vendor/golang.org/x/net/ipv6/icmp_solaris.go +++ b/vendor/golang.org/x/net/ipv6/icmp_solaris.go @@ -5,10 +5,16 @@ package ipv6 func (f *icmpv6Filter) accept(typ ICMPType) { + if typ < 0 || int(typ>>5) >= len(f.X__icmp6_filt) { + return + } f.X__icmp6_filt[typ>>5] |= 1 << (uint32(typ) & 31) } func (f *icmpv6Filter) block(typ ICMPType) { + if typ < 0 || int(typ>>5) >= len(f.X__icmp6_filt) { + return + } f.X__icmp6_filt[typ>>5] &^= 1 << (uint32(typ) & 31) } @@ -23,5 +29,8 @@ } func (f *icmpv6Filter) willBlock(typ ICMPType) bool { + if typ < 0 || int(typ>>5) >= len(f.X__icmp6_filt) { + return false + } return f.X__icmp6_filt[typ>>5]&(1<<(uint32(typ)&31)) == 0 } --- a/vendor/golang.org/x/net/ipv6/icmp_zos.go +++ b/vendor/golang.org/x/net/ipv6/icmp_zos.go @@ -5,11 +5,17 @@ package ipv6 func (f *icmpv6Filter) accept(typ ICMPType) { + if typ < 0 || int(typ>>5) >= len(f.Filt) { + return + } f.Filt[typ>>5] |= 1 << (uint32(typ) & 31) } func (f *icmpv6Filter) block(typ ICMPType) { + if typ < 0 || int(typ>>5) >= len(f.Filt) { + return + } f.Filt[typ>>5] &^= 1 << (uint32(typ) & 31) } @@ -25,5 +31,8 @@ } func (f *icmpv6Filter) willBlock(typ ICMPType) bool { + if typ < 0 || int(typ>>5) >= len(f.Filt) { + return false + } return f.Filt[typ>>5]&(1<<(uint32(typ)&31)) == 0 } --- a/vendor/golang.org/x/net/ipv6/sys_bpf.go +++ b/vendor/golang.org/x/net/ipv6/sys_bpf.go @@ -7,6 +7,8 @@ package ipv6 import ( + "errors" + "math" "unsafe" "golang.org/x/net/bpf" @@ -15,6 +17,9 @@ ) func (so *sockOpt) setAttachFilter(c *socket.Conn, f []bpf.RawInstruction) error { + if len(f) == 0 || len(f) > math.MaxUint16 { + return errors.New("invalid filter") + } prog := unix.SockFprog{ Len: uint16(len(f)), Filter: (*unix.SockFilter)(unsafe.Pointer(&f[0])), --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -345,7 +345,7 @@ # github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible ## explicit github.com/aliyun/aliyun-oss-go-sdk/oss -# github.com/andybalholm/brotli v1.2.6 +# github.com/andybalholm/brotli v1.2.7 ## explicit; go 1.22 github.com/andybalholm/brotli github.com/andybalholm/brotli/matchfinder @@ -391,8 +391,8 @@ github.com/apache/cassandra-gocql-driver/v2/internal/lru github.com/apache/cassandra-gocql-driver/v2/internal/murmur github.com/apache/cassandra-gocql-driver/v2/internal/streams -# github.com/apache/thrift v0.24.0 -## explicit; go 1.25 +# github.com/apache/thrift v0.25.0 +## explicit; go 1.26 github.com/apache/thrift/lib/go/thrift # github.com/armon/go-metrics v0.4.1 ## explicit; go 1.12 @@ -1407,8 +1407,8 @@ ## explicit; go 1.16 github.com/klauspost/asmfmt github.com/klauspost/asmfmt/cmd/asmfmt -# github.com/klauspost/compress v1.18.7 -## explicit; go 1.24 +# github.com/klauspost/compress v1.20.2 +## explicit; go 1.25 github.com/klauspost/compress github.com/klauspost/compress/flate github.com/klauspost/compress/fse @@ -1417,6 +1417,7 @@ github.com/klauspost/compress/internal/cpuinfo github.com/klauspost/compress/internal/le github.com/klauspost/compress/internal/race +github.com/klauspost/compress/internal/regmask github.com/klauspost/compress/internal/snapref github.com/klauspost/compress/s2 github.com/klauspost/compress/snappy @@ -2483,7 +2484,7 @@ golang.org/x/mod/internal/lazyregexp golang.org/x/mod/module golang.org/x/mod/semver -# golang.org/x/net v0.61.0 +# golang.org/x/net v0.62.0 ## explicit; go 1.26.0 golang.org/x/net/bpf golang.org/x/net/context/ctxhttp