From 39d36a680535136905beccc1ed60df49655a2c01 Mon Sep 17 00:00:00 2001 From: Timofey Solomko Date: Sat, 25 Sep 2021 21:44:31 +0300 Subject: [PATCH] [LZ4] Implement block loop and some checks for truncated data --- Sources/LZ4/LZ4.swift | 54 +++++++++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/Sources/LZ4/LZ4.swift b/Sources/LZ4/LZ4.swift index 40643ea4..ceb86dfb 100644 --- a/Sources/LZ4/LZ4.swift +++ b/Sources/LZ4/LZ4.swift @@ -9,14 +9,18 @@ import BitByteData public enum LZ4: DecompressionAlgorithm { public static func decompress(data: Data) throws -> Data { + // Valid LZ4 frame must contain magic number (4 bytes), frame descriptor (at least 3 bytes), and EndMark + // (4 bytes), assuming zero data blocks. + guard data.count >= 11 + else { throw DataError.truncated } let reader = LittleEndianByteReader(data: data) // TODO: Switch between frame and block decoding modes? - // TODO: Small/empty data size check + Tests! (bytesLeft >= 4 + 3 + 4?) + // TODO: Tests for data truncated at various places. // TODO: Test various advanced options of LZ4. - // TODO: Skippable frames - // TODO: Legacy frames // Magic number. + // TODO: Skippable frames + // TODO: Legacy frames guard reader.uint32() == 0x184D2204 else { throw DataError.corrupted } @@ -45,8 +49,13 @@ public enum LZ4: DecompressionAlgorithm { else { throw DataError.corrupted } // Since we don't do manual memory allocation, we don't need to decode the block maximum size from `bd`. - var contentSize: Int? = nil + let contentSize: Int? if contentSizePresent { + // At this point valid LZ4 frame must have at least 13 bytes remaining for: content size (8 bytes), header + // checksum (1 byte), and EndMark (4 bytes), assuming zero data blocks. + // TODO: test truncated + guard reader.bytesLeft >= 13 + else { throw DataError.truncated } // Since Data is indexed by the Int type, the maximum size of the uncompressed data that we can decode is // Int.max. However, LZ4 supports uncompressed data sizes up to UInt64.max, which is larger, so we check // for this possibility. @@ -54,6 +63,8 @@ public enum LZ4: DecompressionAlgorithm { guard rawContentSize <= UInt64(truncatingIfNeeded: Int.max) else { throw DataError.unsupportedFeature } contentSize = Int(truncatingIfNeeded: rawContentSize) + } else { + contentSize = nil } guard !dictIdPresent @@ -65,17 +76,33 @@ public enum LZ4: DecompressionAlgorithm { guard UInt8(truncatingIfNeeded: (headerChecksum >> 8) & 0xFF) == reader.byte() else { throw DataError.corrupted } - // TODO: Data Blocks var out = Data() while true { - out.append(try LZ4.processBlock(reader)) - guard !reader.isFinished - else { throw DataError.truncated } + /// Either the size of the block, or the EndMark. + let blockMark = reader.uint32() // Check for the EndMark. - if reader.uint32() == 0 { + if blockMark == 0 { break + } + // The highest bit indicates if the block is compressed. + let compressed = blockMark & 0x80000000 == 0 + let blockSize = (blockMark & 0x7FFFFFFF).toInt() + // TODO: "Block_Size shall never be larger than Block_Maximum_Size". Should we verify this condition? + // TODO: Check how reference implementation reacts to violation of this condition (during decompression). + + // TODO: test truncated + guard reader.bytesLeft >= blockSize + (blockChecksumPresent ? 4 : 0) + 4 + else { throw DataError.truncated } + + let blockData = data[reader.offset..= 4 + else { throw DataError.truncated } guard XxHash32.hash(data: out) == reader.uint32() else { throw DataError.checksumMismatch([out]) } } return out } - // TODO: Multi-frame decoding (similar to XZArchive.splitUnarchive or GzipArchive.multiUnarchive). + // TODO: Multi-frame decoding, similar to XZArchive.splitUnarchive or GzipArchive.multiUnarchive. - private static func processBlock(_ reader: LittleEndianByteReader) throws -> Data { + private static func processCompressedBlock(_ data: Data) throws -> Data { fatalError("Not implemented yet") }