diff --git a/CHANGELOG.md b/CHANGELOG.md index d621c8e9..60c8c480 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ v3.0.0 - In CocoaPods configurations availability of such support depends on the presence of corresponding podspecs. +v2.4.3 +---------------- +- Fixed incorrect calculation of header's checksum for GZip archives. + v2.4.2 ---------------- - Fixed a problem, where ZipEntry.fileName was returning fileComment instead. diff --git a/SWCompression.xcodeproj/project.pbxproj b/SWCompression.xcodeproj/project.pbxproj index c2353357..efcad866 100644 --- a/SWCompression.xcodeproj/project.pbxproj +++ b/SWCompression.xcodeproj/project.pbxproj @@ -91,6 +91,7 @@ 069DCEA21EB64D0300ADC374 /* test7.answer in Resources */ = {isa = PBXBuildFile; fileRef = 069DCE991EB64D0300ADC374 /* test7.answer */; }; 069DCEA31EB64D0300ADC374 /* test8.answer in Resources */ = {isa = PBXBuildFile; fileRef = 069DCE9A1EB64D0300ADC374 /* test8.answer */; }; 069DCEA41EB64D0300ADC374 /* test9.answer in Resources */ = {isa = PBXBuildFile; fileRef = 069DCE9B1EB64D0300ADC374 /* test9.answer */; }; + 069F27701EDA0D5E00736269 /* TestUnicode.zip in Resources */ = {isa = PBXBuildFile; fileRef = 069F276F1EDA0D5E00736269 /* TestUnicode.zip */; }; 06A393391DE0709300182E12 /* GzipArchive.swift in Sources */ = {isa = PBXBuildFile; fileRef = 063364E21DC52979007E313F /* GzipArchive.swift */; }; 06A3933A1DE0709300182E12 /* ZlibArchive.swift in Sources */ = {isa = PBXBuildFile; fileRef = 064492581DC606D400F10981 /* ZlibArchive.swift */; }; 06A3933B1DE0709300182E12 /* Deflate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 061FCE251DBCC4BE0052F7BE /* Deflate.swift */; }; @@ -205,6 +206,7 @@ 069DCE991EB64D0300ADC374 /* test7.answer */ = {isa = PBXFileReference; lastKnownFileType = file; path = test7.answer; sourceTree = ""; }; 069DCE9A1EB64D0300ADC374 /* test8.answer */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = test8.answer; sourceTree = ""; }; 069DCE9B1EB64D0300ADC374 /* test9.answer */ = {isa = PBXFileReference; lastKnownFileType = file; path = test9.answer; sourceTree = ""; }; + 069F276F1EDA0D5E00736269 /* TestUnicode.zip */ = {isa = PBXFileReference; lastKnownFileType = archive.zip; path = TestUnicode.zip; sourceTree = ""; }; 06A3933E1DE070B500182E12 /* Info-iOS.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = "Info-iOS.plist"; sourceTree = ""; }; 06A3933F1DE070B500182E12 /* Info-tvOS.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = "Info-tvOS.plist"; sourceTree = ""; }; 06A393401DE070B500182E12 /* Info-watchOS.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = "Info-watchOS.plist"; sourceTree = ""; }; @@ -377,6 +379,7 @@ 06BD0D1F1E3D0C630054AA5C /* ZIP */ = { isa = PBXGroup; children = ( + 069F276F1EDA0D5E00736269 /* TestUnicode.zip */, 06BD0D201E3D0C630054AA5C /* SWCompressionSourceCode.zip */, 06BD0D211E3D0C630054AA5C /* TestDataDescriptor.zip */, 06BD0D221E3D0C630054AA5C /* TestZip64.zip */, @@ -433,10 +436,10 @@ 06CC187E1DE35607003532F5 /* DeflateTests.swift */, 0680E1B21DDA3D0A005C05EB /* GzipTests.swift */, 0680E1B01DDA2D7C005C05EB /* ZlibTests.swift */, - 061FCE2C1DBCC6A30052F7BE /* Constants.swift */, 0675183C1E2A883600D16354 /* ZipTests.swift */, - 06BE1AD81DB410F100EE0F59 /* Info.plist */, 0696E5C51ED0B22F00921D4D /* TarTests.swift */, + 061FCE2C1DBCC6A30052F7BE /* Constants.swift */, + 06BE1AD81DB410F100EE0F59 /* Info.plist */, ); path = Tests; sourceTree = ""; @@ -748,6 +751,7 @@ 06CC187D1DE35269003532F5 /* test.zlib in Resources */, 06D07A9B1EB64E7A00BC8C65 /* test3.bz2 in Resources */, 069DCE9C1EB64D0300ADC374 /* test1.answer in Resources */, + 069F27701EDA0D5E00736269 /* TestUnicode.zip in Resources */, 069AC2181E02DE490041AC13 /* test8.lzma in Resources */, 068B36791ED0AC050016269E /* full_test.tar in Resources */, 069DCE8F1EB64B1500ADC374 /* test7.gz in Resources */, diff --git a/Sources/BZip2.swift b/Sources/BZip2.swift index 2eddbde1..898a7db7 100644 --- a/Sources/BZip2.swift +++ b/Sources/BZip2.swift @@ -50,7 +50,7 @@ public enum BZip2Error: Error { } /// Provides function to decompress data, which were compressed using BZip2. -public final class BZip2: DecompressionAlgorithm { +public class BZip2: DecompressionAlgorithm { /** Decompresses `compressedData` with BZip2 algortihm. @@ -94,12 +94,14 @@ public final class BZip2: DecompressionAlgorithm { if blockType == 0x314159265359 { let blockBytes = try decode(data: &pointerData) - guard CheckSums.bzip2CRC32(blockBytes) == blockCRC32 else { throw BZip2Error.wrongCRC(Data(bytes: out)) } + guard CheckSums.bzip2CRC32(blockBytes) == blockCRC32 + else { throw BZip2Error.wrongCRC(Data(bytes: out)) } for byte in blockBytes { out.append(byte) } } else if blockType == 0x177245385090 { - guard CheckSums.bzip2CRC32(out) == blockCRC32 else { throw BZip2Error.wrongCRC(Data(bytes: out)) } + guard CheckSums.bzip2CRC32(out) == blockCRC32 + else { throw BZip2Error.wrongCRC(Data(bytes: out)) } break } else { throw BZip2Error.wrongBlockType @@ -187,7 +189,8 @@ public final class BZip2: DecompressionAlgorithm { } let tables = try computeTables() - var favourites = try used.enumerated().reduce([]) { (partialResult: [UInt8], next: (offset: Int, element: Bool)) throws -> [UInt8] in + var favourites = try used.enumerated().reduce([]) { + (partialResult: [UInt8], next: (offset: Int, element: Bool)) throws -> [UInt8] in if next.element { var newResult = partialResult newResult.append(next.offset.toUInt8()) diff --git a/Sources/BitToByteWriter.swift b/Sources/BitToByteWriter.swift index 93d8c336..7a9c4179 100644 --- a/Sources/BitToByteWriter.swift +++ b/Sources/BitToByteWriter.swift @@ -8,7 +8,7 @@ import Foundation -final class BitToByteWriter { +class BitToByteWriter { private(set) var buffer: [UInt8] = [] private var bitMask: UInt8 diff --git a/Sources/DataWithPointer.swift b/Sources/DataWithPointer.swift index 9355c0b0..0effd558 100644 --- a/Sources/DataWithPointer.swift +++ b/Sources/DataWithPointer.swift @@ -13,7 +13,7 @@ enum BitOrder { case reversed } -final class DataWithPointer { +class DataWithPointer { let bitOrder: BitOrder let size: Int diff --git a/Sources/Deflate.swift b/Sources/Deflate.swift index d0c369a3..504ca309 100644 --- a/Sources/Deflate.swift +++ b/Sources/Deflate.swift @@ -29,7 +29,7 @@ public enum DeflateError: Error { } /// Provides function to decompress data, which were compressed with DEFLATE. -public final class Deflate: DecompressionAlgorithm { +public class Deflate: DecompressionAlgorithm { private struct Constants { static let codeLengthOrders: [Int] = @@ -135,7 +135,8 @@ public final class Deflate: DecompressionAlgorithm { let codeLengthsLength = pointerData.intFromBits(count: 4) + 4 // Read code lengths codes. - // Moreover, they are stored in a very specific order (defined by HuffmanTree.Constants.codeLengthOrders). + // Moreover, they are stored in a very specific order, + // defined by HuffmanTree.Constants.codeLengthOrders. var lengthsForOrder = Array(repeating: 0, count: 19) for i in 0..> 2) - 1) - // Actually, nextSymbol is not a starting value of length but an index for special array of starting values. + // Actually, nextSymbol is not a starting value of length, + // but an index for special array of starting values. let length = Constants.lengthBase[nextSymbol - 257] + pointerData.intFromBits(count: extraLength) @@ -314,7 +318,6 @@ public final class Deflate: DecompressionAlgorithm { // Since `length` of uncompressed block is 16-bit integer, // there is a limitation on size of uncompressed block. // Falling back to static Huffman encoding in case of big uncompressed block is a band-aid solution. - // TODO: Implement spliting uncompressed block into smaller blocks. if uncompBlockSize <= staticHuffmanBlockSize && uncompBlockSize <= 65535 { // If according to our calculations static huffman will not make output smaller than input, // we fallback to creating uncompressed block. @@ -377,18 +380,18 @@ public final class Deflate: DecompressionAlgorithm { for code in bldCodes { switch code { case .byte(let byte): - mainLiterals.code(symbol: byte.toInt(), &bitWriter) + try mainLiterals.code(symbol: byte.toInt(), &bitWriter, DeflateError.symbolNotFound) case .lengthDistance(let ld): - mainLiterals.code(symbol: ld.lengthSymbol, &bitWriter) + try mainLiterals.code(symbol: ld.lengthSymbol, &bitWriter, DeflateError.symbolNotFound) bitWriter.write(number: ld.lengthExtraBits, bitsCount: ld.lengthExtraBitsCount) - mainDistances.code(symbol: ld.distanceSymbol, &bitWriter) + try mainDistances.code(symbol: ld.distanceSymbol, &bitWriter, DeflateError.symbolNotFound) bitWriter.write(number: ld.distanceExtraBits, bitsCount: ld.distanceExtraBitsCount) } } // End data symbol. - mainLiterals.code(symbol: 256, &bitWriter) + try mainLiterals.code(symbol: 256, &bitWriter, DeflateError.symbolNotFound) bitWriter.finish() return bitWriter.buffer @@ -432,7 +435,8 @@ public final class Deflate: DecompressionAlgorithm { case .byte(let byte): return "raw symbol: \(byte)" case .lengthDistance(let ld): - return "length: \(ld.length), length symbol: \(ld.lengthSymbol), distance: \(ld.distance), distance symbol: \(ld.distanceSymbol)" + return "length: \(ld.length), length symbol: \(ld.lengthSymbol), " + + "distance: \(ld.distance), distance symbol: \(ld.distanceSymbol)" } } } diff --git a/Sources/GzipArchive.swift b/Sources/GzipArchive.swift index 04158b0b..1bcedd43 100644 --- a/Sources/GzipArchive.swift +++ b/Sources/GzipArchive.swift @@ -37,6 +37,8 @@ public enum GzipError: Error { case wrongCRC(Data) /// Computed isize didn't match the value stored in the archive. case wrongISize + + case cannotEncodeISOLatin1 } /// A structure which provides information about gzip archive. @@ -66,7 +68,7 @@ public struct GzipHeader { case ntfs = 11 /// File system was unknown to the archiver. case unknown = 255 - /// File system was one of the rare systems.. + /// File system was one of the rare systems. case other = 256 } @@ -81,6 +83,8 @@ public struct GzipHeader { /// Comment inside the archive. public let comment: String? + public let isTextFile: Bool + /** Initializes the structure with the values of first 'member' in gzip archive presented in `archiveData`. @@ -91,7 +95,6 @@ public struct GzipHeader { - Throws: `GzipError`. It may indicate that either the data is damaged or it might not be compressed with gzip at all. */ - public init(archiveData: Data) throws { let pointerData = DataWithPointer(data: archiveData, bitOrder: .reversed) try self.init(pointerData) @@ -126,6 +129,8 @@ public struct GzipHeader { self.osType = FileSystemType(rawValue: pointerData.alignedByte().toInt()) ?? .other headerBytes.append(self.osType.rawValue.toUInt8()) + self.isTextFile = flags & Flags.ftext != 0 + // Some archives may contain extra fields if flags & Flags.fextra != 0 { let xlen = pointerData.intFromAlignedBytes(count: 2) @@ -142,9 +147,9 @@ public struct GzipHeader { var fnameBytes: [UInt8] = [] while true { let byte = pointerData.alignedByte() + headerBytes.append(byte) guard byte != 0 else { break } fnameBytes.append(byte) - headerBytes.append(byte) } self.originalFileName = String(data: Data(fnameBytes), encoding: .utf8) } else { @@ -156,9 +161,9 @@ public struct GzipHeader { var fcommentBytes: [UInt8] = [] while true { let byte = pointerData.alignedByte() + headerBytes.append(byte) guard byte != 0 else { break } fcommentBytes.append(byte) - headerBytes.append(byte) } self.comment = String(data: Data(fcommentBytes), encoding: .utf8) } else { @@ -177,7 +182,7 @@ public struct GzipHeader { } /// Provides unarchive function for GZip archives. -public final class GzipArchive: Archive { +public class GzipArchive: Archive { /** Unarchives gzip archive stored in `archiveData`. @@ -199,23 +204,17 @@ public final class GzipArchive: Archive { /// Object with input data which supports convenient work with bit shifts. var pointerData = DataWithPointer(data: data, bitOrder: .reversed) - var out: [UInt8] = [] + _ = try GzipHeader(pointerData) - while !pointerData.isAtTheEnd { - _ = try GzipHeader(pointerData) + let memberData = Data(bytes: try Deflate.decompress(&pointerData)) - let memberData = try Deflate.decompress(&pointerData) + let crc32 = pointerData.uint32FromAlignedBytes(count: 4) + guard CheckSums.crc32(memberData) == crc32 else { throw GzipError.wrongCRC(memberData) } - let crc32 = pointerData.uint32FromAlignedBytes(count: 4) - guard CheckSums.crc32(memberData) == crc32 else { throw GzipError.wrongCRC(Data(bytes: out)) } + let isize = pointerData.intFromAlignedBytes(count: 4) + guard UInt64(memberData.count) % UInt64(1) << 32 == UInt64(isize) else { throw GzipError.wrongISize } - let isize = pointerData.intFromAlignedBytes(count: 4) - guard UInt64(memberData.count) % UInt64(1) << 32 == UInt64(isize) else { throw GzipError.wrongISize } - - out.append(contentsOf: memberData) - } - - return Data(bytes: out) + return memberData } /** @@ -234,16 +233,81 @@ public final class GzipArchive: Archive { - Returns: Data object with resulting archive. */ - public static func archive(data: Data) throws -> Data { - let out: [UInt8] = [ + public static func archive(data: Data, comment: String? = nil, fileName: String? = nil, + writeHeaderCRC: Bool = false, isTextFile: Bool = false, + osType: GzipHeader.FileSystemType? = nil, modificationTime: Date? = nil) throws -> Data { + var flags: UInt8 = 0 + + var commentData = Data() + if var comment = comment { + flags |= 1 << 4 + if comment.characters.last != "\u{00}" { + comment.append("\u{00}") + } + if let data = comment.data(using: .isoLatin1) { + commentData = data + } else { + throw GzipError.cannotEncodeISOLatin1 + } + } + + var fileNameData = Data() + if var fileName = fileName { + flags |= 1 << 3 + if fileName.characters.last != "\u{00}" { + fileName.append("\u{00}") + } + if let data = fileName.data(using: .isoLatin1) { + fileNameData = data + } else { + throw GzipError.cannotEncodeISOLatin1 + } + } + + if writeHeaderCRC { + flags |= 1 << 1 + } + + if isTextFile { + flags |= 1 << 0 + } + + var os: UInt8 = 255 + if let osType = osType { + os = (osType == .other ? 255 : osType.rawValue).toUInt8() + } + + var mtimeBytes: [UInt8] = [0, 0, 0, 0] + if let modificationTime = modificationTime { + let timeInterval = Int(modificationTime.timeIntervalSince1970) + for i in 0..<4 { + mtimeBytes[i] = UInt8(truncatingBitPattern: (timeInterval & (0xFF << (i * 8))) >> (i * 8)) + } + } + + var headerBytes: [UInt8] = [ 0x1f, 0x8b, // 'magic' bytes. 8, // Compression method (DEFLATE). - 0, // Flags; currently no flags are set. - 0, 0, 0, 0, // Mtime; currently timestamp is not set. - 2, // Extra flags; 2 means that DEFLATE used slowest algorithm. - 255, // OS Type; set to default value (unknown). + flags // Flags; currently no flags are set. ] - var outData = Data(bytes: out) + for i in 0..<4 { + headerBytes.append(mtimeBytes[i]) + } + headerBytes.append(2) // Extra flags; 2 means that DEFLATE used slowest algorithm. + headerBytes.append(os) + + var outData = Data(bytes: headerBytes) + + outData.append(fileNameData) + outData.append(commentData) + + if writeHeaderCRC { + let headerCRC = CheckSums.crc32(outData) + for i: UInt32 in 0..<2 { + outData.append(UInt8((headerCRC & (0xFF << (i * 8))) >> (i * 8))) + } + } + outData.append(try Deflate.compress(data: data)) let crc32 = CheckSums.crc32(data) diff --git a/Sources/HuffmanTree.swift b/Sources/HuffmanTree.swift index 29422f3c..01ad47ca 100644 --- a/Sources/HuffmanTree.swift +++ b/Sources/HuffmanTree.swift @@ -156,8 +156,8 @@ class HuffmanTree { } } - func code(symbol: Int, _ bitWriter: inout BitToByteWriter) { - precondition(self.coding, "Tree is not initalized for coding!") + func code(symbol: Int, _ bitWriter: inout BitToByteWriter, _ symbolNotFoundError: Error) throws { + precondition(self.coding, "HuffmanTree is not initalized for coding!") var index = 0 while true { @@ -166,9 +166,9 @@ class HuffmanTree { if foundSymbol == symbol { return } else { - fatalError("Symbol not found, this error should be replaced with Error.") + throw symbolNotFoundError } - case .branch(_): + case .branch: let leftChildIndex = 2 * index + 1 if leftChildIndex < self.leafCount { switch self.tree[leftChildIndex] { @@ -200,7 +200,7 @@ class HuffmanTree { bitWriter.write(bit: 1) continue } else { - fatalError("Symbol not found, this error should be replaced with Error.") + throw symbolNotFoundError } case .branch(let rightArray): if rightArray.contains(symbol) { @@ -208,7 +208,7 @@ class HuffmanTree { bitWriter.write(bit: 1) continue } else { - fatalError("Symbol not found, this error should be replaced with Error.") + throw symbolNotFoundError } } } diff --git a/Sources/LZMA.swift b/Sources/LZMA.swift index 1a5e03c9..b1d7fb12 100644 --- a/Sources/LZMA.swift +++ b/Sources/LZMA.swift @@ -38,10 +38,12 @@ public enum LZMAError: Error { case repeatWillExceed /// The amount of already decoded bytes is smaller than repeat length. case notEnoughToRepeat + + case decoderIsNotInitialised } /// Provides function to decompress data, which were compressed with LZMA -public final class LZMA: DecompressionAlgorithm { +public class LZMA: DecompressionAlgorithm { /** Decompresses `compressedData` with LZMA algortihm. @@ -63,28 +65,8 @@ public final class LZMA: DecompressionAlgorithm { } static func decompress(_ pointerData: inout DataWithPointer) throws -> [UInt8] { - // Firstly, we need to parse LZMA properties. - var properties = pointerData.alignedByte() - if properties >= (9 * 5 * 5) { - throw LZMAError.wrongProperties - } - /// The number of literal context bits - let lc = properties % 9 - properties /= 9 - /// The number of pos bits - let pb = properties / 5 - /// The number of literal pos bits - let lp = properties % 5 - var dictionarySize = pointerData.intFromAlignedBytes(count: 4) - dictionarySize = dictionarySize < (1 << 12) ? 1 << 12 : dictionarySize - - /// Size of uncompressed data. -1 means it is unknown/undefined. - var uncompressedSize = pointerData.intFromAlignedBytes(count: 8) - uncompressedSize = Double(uncompressedSize) == pow(Double(2), Double(64)) - 1 ? -1 : uncompressedSize - - let lzmaDecoder = try LZMADecoder(&pointerData, lc, pb, lp, dictionarySize) - - try lzmaDecoder.decodeLZMA(&uncompressedSize) + let lzmaDecoder = try LZMADecoder(&pointerData) + try lzmaDecoder.decodeLZMA() return lzmaDecoder.out } diff --git a/Sources/LZMA2.swift b/Sources/LZMA2.swift index 9dba060d..d4995bf6 100644 --- a/Sources/LZMA2.swift +++ b/Sources/LZMA2.swift @@ -35,7 +35,7 @@ public enum LZMA2Error: Error { } /// Provides function to decompress data, which were compressed with LZMA2 -public final class LZMA2: DecompressionAlgorithm { +public class LZMA2: DecompressionAlgorithm { /** Decompresses `compressedData` with LZMA2 algortihm. LZMA2 is a modification of LZMA. @@ -60,7 +60,7 @@ public final class LZMA2: DecompressionAlgorithm { static func decompress(_ dictionarySize: Int, _ pointerData: inout DataWithPointer) throws -> [UInt8] { // At this point lzmaDecoder will be in a VERY bad state. - let lzmaDecoder = try LZMADecoder(&pointerData, 0, 0, 0, 0) + let lzmaDecoder = try LZMADecoder(&pointerData) try lzmaDecoder.decodeLZMA2(dictionarySize) return lzmaDecoder.out } diff --git a/Sources/LZMABitTreeDecoder.swift b/Sources/LZMABitTreeDecoder.swift index cd1490f6..997e541c 100644 --- a/Sources/LZMABitTreeDecoder.swift +++ b/Sources/LZMABitTreeDecoder.swift @@ -9,7 +9,7 @@ import Foundation /// Used to decode symbols that need several bits for storing. -final class LZMABitTreeDecoder { +class LZMABitTreeDecoder { private var pointerData: DataWithPointer diff --git a/Sources/LZMADecoder.swift b/Sources/LZMADecoder.swift index eaccf214..0b7de811 100644 --- a/Sources/LZMADecoder.swift +++ b/Sources/LZMADecoder.swift @@ -24,16 +24,16 @@ struct LZMAConstants { // LZMAConstants.numStates << LZMAConstants.numPosBitsMax = 192 } -final class LZMADecoder { +class LZMADecoder { private var pointerData: DataWithPointer - private var lc: UInt8 - private var lp: UInt8 - private var pb: UInt8 - private var dictionarySize: Int + private var lc: UInt8 = 0 + private var lp: UInt8 = 0 + private var pb: UInt8 = 0 + private var dictionarySize: Int = 0 - private var rangeDecoder: LZMARangeDecoder + private var rangeDecoder: LZMARangeDecoder = LZMARangeDecoder() private var posSlotDecoder: [LZMABitTreeDecoder] = [] private var alignDecoder: LZMABitTreeDecoder private var lenDecoder: LZMALenDecoder @@ -43,7 +43,7 @@ final class LZMADecoder { For literal decoding we need `1 << (lc + lp)` amount of tables. Each table contains 0x300 probabilities. */ - private var literalProbs: [[Int]] + private var literalProbs: [[Int]] = [] /** Array with all probabilities: @@ -55,9 +55,9 @@ final class LZMADecoder { - 229..<241: isRepG2 - 241..<433: isRep0Long */ - private var probabilities: [Int] = Array(repeating: LZMAConstants.probInitValue, count: 2 * 192 + 4 * 12) + private var probabilities: [Int] = [] - private var posDecoders: [Int] + private var posDecoders: [Int] = [] // 'Distance history table'. private var rep0: Int = 0 @@ -74,29 +74,13 @@ final class LZMADecoder { private var dictStart = 0 private var dictEnd = 0 - init(_ pointerData: inout DataWithPointer, _ lc: UInt8, _ pb: UInt8, _ lp: UInt8, - _ dictionarySize: Int) throws { + /// For proper processing of LZMA data `resetState` and `resetProperties` functions should be called at least once. + /// If that has happened, then stateReset is true. + private var stateReset: Bool = false + + init(_ pointerData: inout DataWithPointer) throws { self.pointerData = pointerData - - self.lc = lc - self.lp = lp - self.pb = pb - self.dictionarySize = dictionarySize - - self.rangeDecoder = LZMARangeDecoder() - - self.literalProbs = Array(repeating: Array(repeating: LZMAConstants.probInitValue, - count: 0x300), - count: 1 << (lc + lp).toInt()) - - self.posSlotDecoder = [] - for _ in 0..= (9 * 5 * 5) { @@ -116,11 +105,9 @@ final class LZMADecoder { self.pb = properties / 5 /// The number of literal pos bits self.lp = properties % 5 - } - private func resetDictionary(_ dictSize: Int) { - self.dictionarySize = dictSize - self.dictStart = self.dictEnd + // We need to 'reset state' because several properties of Decoder depend on the values of lc, lp, pb. + self.resetState() } private func resetState() { @@ -145,17 +132,15 @@ final class LZMADecoder { count: 1 + LZMAConstants.numFullDistances - LZMAConstants.endPosModelIndex) self.lenDecoder = LZMALenDecoder(&self.pointerData) self.repLenDecoder = LZMALenDecoder(&self.pointerData) + + self.stateReset = true } private func decodeUncompressed() { let dataSize = self.pointerData.alignedByte().toInt() << 8 + self.pointerData.alignedByte().toInt() + 1 for _ in 0..> 7) & 1).toInt() matchByte <<= 1 - let bit = rangeDecoder.decode(bitWithProb: &literalProbs[litState][((1 + matchBit) << 8) + symbol]) + let bit = rangeDecoder.decode(bitWithProb: + &literalProbs[litState][((1 + matchBit) << 8) + symbol]) symbol = (symbol << 1) | bit if matchBit != bit { break @@ -292,12 +296,13 @@ final class LZMADecoder { if dictEnd == 0 { throw LZMAError.windowIsEmpty } if rangeDecoder.decode(bitWithProb: &probabilities[205 + state]) == 0 { // (We use last distance from 'distance history table'). - if rangeDecoder.decode(bitWithProb: &probabilities[241 + (state << LZMAConstants.numPosBitsMax) + posState]) == 0 { + if rangeDecoder.decode(bitWithProb: + &probabilities[241 + (state << LZMAConstants.numPosBitsMax) + posState]) == 0 { // SHORT REP MATCH CASE state = state < 7 ? 9 : 11 let byte = self.byte(at: rep0 + 1) - uncompressedSize -= 1 self.put(byte) + uncompressedSize -= 1 continue } } else { // REP MATCH CASE @@ -373,15 +378,17 @@ final class LZMADecoder { } if uncompressedSize == 0 { throw LZMAError.exceededUncompressedSize } - if rep0 >= dictionarySize || (rep0 > dictEnd && dictEnd < dictionarySize) { throw LZMAError.notEnoughToRepeat } + if rep0 >= dictionarySize || (rep0 > dictEnd && dictEnd < dictionarySize) { + throw LZMAError.notEnoughToRepeat + } } // Converting from zero-based length of the match to the real one. len += LZMAConstants.matchMinLen if uncompressedSize > -1 && uncompressedSize < len { throw LZMAError.repeatWillExceed } for _ in 0.. (checkType: Int, flagsCRC: UInt32) { + private static func processStreamHeader(_ pointerData: inout DataWithPointer) throws -> (checkType: UInt8, flagsCRC: UInt32) { // Check magic number. guard pointerData.uint64FromAlignedBytes(count: 6) == 0x005A587A37FD else { throw XZError.wrongMagic } - // First byte of flags must be equal to zero. - guard pointerData.alignedByte() == 0 + let flagsBytes = pointerData.alignedBytes(count: 2) + + // First, we need to check for corruption in flags, + // so we compare CRC32 of flags to the value stored in archive. + let flagsCRC = pointerData.uint32FromAlignedBytes(count: 4) + guard CheckSums.crc32(flagsBytes) == flagsCRC + else { throw XZError.wrongInfoCRC } + + // If data is not corrupted, then some bits must be equal to zero. + guard flagsBytes[0] == 0 && flagsBytes[1] & 0xF0 == 0 else { throw XZError.fieldReservedValue } - // Next four bits indicate type of redundancy check. - let checkType = pointerData.intFromBits(count: 4) + // Four bits of second flags byte indicate type of redundancy check. + let checkType = flagsBytes[1] & 0x0F switch checkType { case 0x00, 0x01, 0x04, 0x0A: break @@ -160,20 +199,11 @@ public final class XZArchive: Archive { throw XZError.fieldReservedValue } - // Final four bits must be equal to zero. - guard pointerData.intFromBits(count: 4) == 0 - else { throw XZError.fieldReservedValue } - - // CRC-32 of flags must be equal to the value in archive. - let flagsCRC = pointerData.uint32FromAlignedBytes(count: 4) - guard CheckSums.crc32([0, checkType.toUInt8()]) == flagsCRC - else { throw XZError.wrongInfoCRC } - return (checkType, flagsCRC) } private static func processBlock(_ blockHeaderSize: UInt8, - _ pointerData: inout DataWithPointer) throws -> (blockData: [UInt8], unpaddedSize: Int, uncompressedSize: Int) { + _ pointerData: inout DataWithPointer) throws -> (blockData: [UInt8], unpaddedSize: Int, uncompressedSize: Int) { var blockBytes: [UInt8] = [] let blockHeaderStartIndex = pointerData.index - 1 blockBytes.append(blockHeaderSize) @@ -311,7 +341,7 @@ public final class XZArchive: Archive { return indexBytes.count + 4 } - private static func processFooter(_ streamHeader: (checkType: Int, flagsCRC: UInt32), + private static func processFooter(_ streamHeader: (checkType: UInt8, flagsCRC: UInt32), _ indexSize: Int, _ pointerData: inout DataWithPointer) throws { let footerCRC = pointerData.uint32FromAlignedBytes(count: 4) @@ -333,7 +363,7 @@ public final class XZArchive: Archive { // Flags in the footer should be the same as in the header. guard footerStreamFlags[0] == 0 else { throw XZError.fieldReservedValue } - guard footerStreamFlags[1] & 0x0F == streamHeader.checkType.toUInt8() + guard footerStreamFlags[1] & 0x0F == streamHeader.checkType else { throw XZError.wrongArchiveInfo } guard footerStreamFlags[1] & 0xF0 == 0 else { throw XZError.fieldReservedValue } diff --git a/Sources/ZipContainer.swift b/Sources/ZipContainer.swift index f49b3a2a..e5ed551f 100644 --- a/Sources/ZipContainer.swift +++ b/Sources/ZipContainer.swift @@ -81,7 +81,14 @@ public class ZipEntry: ContainerEntry { Particularly, it is true if size of data is 0 and last character of entry's name is '/'. */ public var isDirectory: Bool { - return self.size == 0 && self.name.characters.last == "/" + let hostSystem = (cdEntry.versionMadeBy & 0xFF00) >> 8 + if hostSystem == 0 || hostSystem == 3 { // MS-DOS or UNIX case. + // In both of this cases external file attributes indicate if this is a directory. + // This is indicated by a special bit in the lowest byte of attributes. + return cdEntry.externalFileAttributes & 0x10 != 0 + } else { + return size == 0 && name.characters.last == "/" + } } /** @@ -155,9 +162,9 @@ public class ZipEntry: ContainerEntry { } // Now, let's update from CD with values from data descriptor. crc32 = pointerData.uint32FromAlignedBytes(count: 4) - compSize = Int(pointerData.uint32FromAlignedBytes(count: 4)) - uncompSize = Int(pointerData.uint32FromAlignedBytes(count: 4)) - // TODO: It may be ZIP64 Data Descriptor. + let sizeOfSizeField: UInt32 = localHeader.zip64FieldsArePresent ? 8 : 4 + compSize = Int(pointerData.uint32FromAlignedBytes(count: sizeOfSizeField)) + uncompSize = Int(pointerData.uint32FromAlignedBytes(count: sizeOfSizeField)) } guard compSize == realCompSize && uncompSize == fileBytes.count @@ -168,7 +175,7 @@ public class ZipEntry: ContainerEntry { return Data(bytes: fileBytes) } - init(_ cdEntry: CentralDirectoryEntry, _ pointerData: inout DataWithPointer) { + fileprivate init(_ cdEntry: CentralDirectoryEntry, _ pointerData: inout DataWithPointer) { self.cdEntry = cdEntry self.pointerData = pointerData } @@ -186,9 +193,10 @@ public class ZipContainer: Container { It is likely that directories will be encountered earlier than files stored in those directories, but one SHOULD NOT assume that this is the case. - - Note: Currently, there is no universal (platform and file system independent) method to determine if entry is a directory. - One can check this by looking at the size of entry's data (it should be 0 for directory) AND - the last character of entry's name (it should be '/'). If all of these is true then entry is likely to be a directory. + - Note: Currently, there is no universal (platform and file system independent) method to determine, + if entry is a directory. One can check this by looking at the size of entry's data + (it should be 0 for directory) AND the last character of entry's name (it should be '/'). + If all of these is true then entry is likely to be a directory. - Parameter containerData: Data of ZIP container. @@ -245,6 +253,8 @@ struct LocalHeader { private(set) var compSize: UInt64 private(set) var uncompSize: UInt64 + private(set) var zip64FieldsArePresent: Bool = false + let fileName: String init(_ pointerData: inout DataWithPointer) throws { @@ -286,6 +296,8 @@ struct LocalHeader { // In local header both uncompressed size and compressed size fields are required. self.uncompSize = pointerData.uint64FromAlignedBytes(count: 8) self.compSize = pointerData.uint64FromAlignedBytes(count: 8) + + self.zip64FieldsArePresent = true default: pointerData.index += size } @@ -452,7 +464,10 @@ struct CentralDirectoryEntry { struct EndOfCentralDirectory { + /// Number of the current disk. private(set) var currentDiskNumber: UInt32 + + /// Number of the disk with the start of CD. private(set) var cdDiskNumber: UInt32 private(set) var cdEntries: UInt64 private(set) var cdSize: UInt64 @@ -462,9 +477,7 @@ struct EndOfCentralDirectory { /// Indicates if Zip64 records should be present. var zip64RecordExists = false - /// Number of current disk. self.currentDiskNumber = pointerData.uint32FromAlignedBytes(count: 2) - /// Number of the disk with the start of CD. self.cdDiskNumber = pointerData.uint32FromAlignedBytes(count: 2) guard self.currentDiskNumber == self.cdDiskNumber else { throw ZipError.multiVolumesNotSupported } diff --git a/Sources/ZlibArchive.swift b/Sources/ZlibArchive.swift index 73a1e77b..2b280b28 100644 --- a/Sources/ZlibArchive.swift +++ b/Sources/ZlibArchive.swift @@ -122,7 +122,7 @@ public struct ZlibHeader { } /// Provides unarchive function for Zlib archives. -public final class ZlibArchive: Archive { +public class ZlibArchive: Archive { /** Unarchives Zlib archive stored in `archiveData`. @@ -171,7 +171,7 @@ public final class ZlibArchive: Archive { public static func archive(data: Data) throws -> Data { let out: [UInt8] = [ 120, // CM (Compression Method) = 8 (DEFLATE), CINFO (Compression Info) = 7 (32K window size). - 218, // Flags: slowest algorithm, no preset dictionary. + 218 // Flags: slowest algorithm, no preset dictionary. ] var outData = Data(bytes: out) outData.append(try Deflate.compress(data: data)) diff --git a/Tests/GzipTests.swift b/Tests/GzipTests.swift index 7cd4504f..fc17e48b 100644 --- a/Tests/GzipTests.swift +++ b/Tests/GzipTests.swift @@ -44,9 +44,7 @@ class GzipTests: XCTestCase { } // Test GZip unarchiving. - let decompressedData = try? GzipArchive.unarchive(archiveData: archiveData) - - guard decompressedData != nil else { + guard let decompressedData = try? GzipArchive.unarchive(archiveData: archiveData) else { XCTFail("Failed to decompress") return } @@ -66,6 +64,51 @@ class GzipTests: XCTestCase { #endif } + func archive(test testName: String) { + // Load answer data. + guard let answerData = try? Data(contentsOf: Constants.url(forAnswer: testName)) else { + XCTFail("Failed to get the answer") + return + } + + // Options for archiving. + let mtimeDate = Date(timeIntervalSinceNow: 0.0) + let mtime = Double(Int(mtimeDate.timeIntervalSince1970)) + + // Test GZip archiving. + guard let archiveData = try? GzipArchive.archive(data: answerData, + comment: "some file comment", + fileName: testName + ".answer", + writeHeaderCRC: true, + isTextFile: true, + osType: .macintosh, + modificationTime: mtimeDate) else { + XCTFail("Failed to create archive.") + return + } + + // Test output GZip header. + guard let testGzipHeader = try? GzipHeader(archiveData: archiveData) else { + XCTFail("Failed to get archive header") + return + } + + XCTAssertEqual(testGzipHeader.compressionMethod, .deflate, "Incorrect compression method") + XCTAssertEqual(testGzipHeader.modificationTime?.timeIntervalSince1970, mtime, "Incorrect mtime") + XCTAssertEqual(testGzipHeader.osType, .macintosh, "Incorrect os type") + XCTAssertEqual(testGzipHeader.originalFileName, "\(testName).answer", "Incorrect original file name") + XCTAssertEqual(testGzipHeader.comment, "some file comment", "Incorrect comment") + XCTAssertTrue(testGzipHeader.isTextFile) + + // Test output GZip archive content. + guard let decompressedData = try? GzipArchive.unarchive(archiveData: archiveData) else { + XCTFail("Failed to decompress") + return + } + + XCTAssertEqual(decompressedData, answerData, "Decompression was incorrect") + } + func testGzip1() { self.header(test: "test1", mtime: 1482698300) self.unarchive(test: "test1") @@ -111,4 +154,8 @@ class GzipTests: XCTestCase { self.unarchive(test: "test9") } + func testGzipArchive4() { + self.archive(test: "test4") + } + } diff --git a/Tests/TarTests.swift b/Tests/TarTests.swift index 8d320fcf..227b36ea 100644 --- a/Tests/TarTests.swift +++ b/Tests/TarTests.swift @@ -48,7 +48,10 @@ class TarTests: XCTestCase { XCTAssertEqual(result.count, 5) for entry in result { - let tarEntry = entry as! TarEntry + guard let tarEntry = entry as? TarEntry else { + XCTFail("Unable to convert entry to TarEntry.") + return + } let name = tarEntry.name.components(separatedBy: ".")[0] guard let answerData = try? Data(contentsOf: Constants.url(forAnswer: name)) else { XCTFail("Failed to get the answer") @@ -56,7 +59,7 @@ class TarTests: XCTestCase { } XCTAssertEqual(tarEntry.data(), answerData) XCTAssertEqual(tarEntry.isDirectory, false) - XCTAssert(tarEntry.accessTime != nil) + XCTAssertNotNil(tarEntry.accessTime) } } diff --git a/Tests/Test Files/XZ/test4.xz b/Tests/Test Files/XZ/test4.xz index 7e111d05..6e48fd6d 100644 --- a/Tests/Test Files/XZ/test4.xz +++ b/Tests/Test Files/XZ/test4.xz @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9e7d47eb06f830a510511db4480a355a82428a324035be9ca24926537b281fed -size 96 +oid sha256:11bd10a5b044b914097caaf218b57258d693f7c4943c52e6b8602267abe69490 +size 100 diff --git a/Tests/Test Files/ZIP/TestUnicode.zip b/Tests/Test Files/ZIP/TestUnicode.zip new file mode 100644 index 00000000..1a216ac7 --- /dev/null +++ b/Tests/Test Files/ZIP/TestUnicode.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcbef95b205c0ef54a4c4da590429b2b0ae4ef334476dac5abe971cf59d0542f +size 340 diff --git a/Tests/XzTests.swift b/Tests/XzTests.swift index 02f0bb4f..b7227b56 100644 --- a/Tests/XzTests.swift +++ b/Tests/XzTests.swift @@ -56,6 +56,7 @@ class XZTests: XCTestCase { } func testXz4() { + // This test contains padding! self.perform(test: "test4") } diff --git a/Tests/ZipTests.swift b/Tests/ZipTests.swift index 8aed00fe..f5e97c26 100644 --- a/Tests/ZipTests.swift +++ b/Tests/ZipTests.swift @@ -69,11 +69,31 @@ class ZipTests: XCTestCase { return } - for entry in entries { - if !entry.isDirectory { - XCTAssertNotNil(try? entry.data()) - } + for entry in entries where !entry.isDirectory { + XCTAssertNotNil(try? entry.data()) } } + func testUnicode() { + guard let testData = try? Data(contentsOf: Constants.url(forTest: "TestUnicode", + withType: ZipTests.testType), + options: .mappedIfSafe) else { + XCTFail("Failed to load test archive") + return + } + + guard let entries = try? ZipContainer.open(containerData: testData) else { + XCTFail("Unable to open ZIP archive.") + return + } + + guard entries.count == 1 else { + XCTFail("Incorrect number of entries.") + return + } + + XCTAssertEqual(entries[0].name, "текстовый файл") + XCTAssertEqual(entries[0].isDirectory, false) + } + }