[LZ4] Implement block loop and some checks for truncated data

This commit is contained in:
Timofey Solomko committed 2021-09-25 21:44:53 +03:00
1 parent bb39e6e36d
commit 39d36a6805
1 file changed
+42 -12
+42 -12
View File
@@ -9,14 +9,18 @@ import BitByteData
public enum LZ4: DecompressionAlgorithm {
public static func decompress(data: Data) throws -> Data {
// Valid LZ4 frame must contain magic number (4 bytes), frame descriptor (at least 3 bytes), and EndMark
// (4 bytes), assuming zero data blocks.
guard data.count >= 11
else { throw DataError.truncated }
let reader = LittleEndianByteReader(data: data)
// TODO: Switch between frame and block decoding modes?
// TODO: Small/empty data size check + Tests! (bytesLeft >= 4 + 3 + 4?)
// TODO: Tests for data truncated at various places.
// TODO: Test various advanced options of LZ4.
// TODO: Skippable frames
// TODO: Legacy frames
// Magic number.
// TODO: Skippable frames
// TODO: Legacy frames
guard reader.uint32() == 0x184D2204
else { throw DataError.corrupted }
@@ -45,8 +49,13 @@ public enum LZ4: DecompressionAlgorithm {
else { throw DataError.corrupted }
// Since we don't do manual memory allocation, we don't need to decode the block maximum size from `bd`.
var contentSize: Int? = nil
let contentSize: Int?
if contentSizePresent {
// At this point valid LZ4 frame must have at least 13 bytes remaining for: content size (8 bytes), header
// checksum (1 byte), and EndMark (4 bytes), assuming zero data blocks.
// TODO: test truncated
guard reader.bytesLeft >= 13
else { throw DataError.truncated }
// Since Data is indexed by the Int type, the maximum size of the uncompressed data that we can decode is
// Int.max. However, LZ4 supports uncompressed data sizes up to UInt64.max, which is larger, so we check
// for this possibility.
@@ -54,6 +63,8 @@ public enum LZ4: DecompressionAlgorithm {
guard rawContentSize <= UInt64(truncatingIfNeeded: Int.max)
else { throw DataError.unsupportedFeature }
contentSize = Int(truncatingIfNeeded: rawContentSize)
} else {
contentSize = nil
}
guard !dictIdPresent
@@ -65,17 +76,33 @@ public enum LZ4: DecompressionAlgorithm {
guard UInt8(truncatingIfNeeded: (headerChecksum >> 8) & 0xFF) == reader.byte()
else { throw DataError.corrupted }
// TODO: Data Blocks
var out = Data()
while true {
out.append(try LZ4.processBlock(reader))
guard !reader.isFinished
else { throw DataError.truncated }
/// Either the size of the block, or the EndMark.
let blockMark = reader.uint32()
// Check for the EndMark.
if reader.uint32() == 0 {
if blockMark == 0 {
break
}
// The highest bit indicates if the block is compressed.
let compressed = blockMark & 0x80000000 == 0
let blockSize = (blockMark & 0x7FFFFFFF).toInt()
// TODO: "Block_Size shall never be larger than Block_Maximum_Size". Should we verify this condition?
// TODO: Check how reference implementation reacts to violation of this condition (during decompression).
// TODO: test truncated
guard reader.bytesLeft >= blockSize + (blockChecksumPresent ? 4 : 0) + 4
else { throw DataError.truncated }
let blockData = data[reader.offset..<reader.offset + blockSize]
reader.offset += blockSize
guard !blockChecksumPresent || XxHash32.hash(data: blockData) == reader.uint32()
else { throw DataError.corrupted }
if compressed {
out.append(try LZ4.processCompressedBlock(blockData))
} else {
reader.offset -= 4
out.append(blockData)
}
}
if contentSizePresent {
@@ -83,15 +110,18 @@ public enum LZ4: DecompressionAlgorithm {
else { throw DataError.corrupted }
}
if contentChecksumPresent {
// TODO: test truncated
guard reader.bytesLeft >= 4
else { throw DataError.truncated }
guard XxHash32.hash(data: out) == reader.uint32()
else { throw DataError.checksumMismatch([out]) }
}
return out
}
// TODO: Multi-frame decoding (similar to XZArchive.splitUnarchive or GzipArchive.multiUnarchive).
// TODO: Multi-frame decoding, similar to XZArchive.splitUnarchive or GzipArchive.multiUnarchive.
private static func processBlock(_ reader: LittleEndianByteReader) throws -> Data {
private static func processCompressedBlock(_ data: Data) throws -> Data {
fatalError("Not implemented yet")
}