From 41b77775e1e31d20e759e868b9040fc0b6f71745 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Mon, 18 Sep 2023 09:54:14 +0200 Subject: [PATCH] Add RedisHashSlot (#91) --- NOTICE.txt | 99 +++----- Sources/RediStack/RedisHashSlot.swift | 222 ++++++++++++++++++ Tests/RediStackTests/RedisHashSlotTests.swift | 78 ++++++ 3 files changed, 332 insertions(+), 67 deletions(-) create mode 100644 Sources/RediStack/RedisHashSlot.swift create mode 100644 Tests/RediStackTests/RedisHashSlotTests.swift diff --git a/NOTICE.txt b/NOTICE.txt index ea02364..cad1fac 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -26,7 +26,7 @@ components that this product depends on. ------------------------------------------------------------------------------- -This product was heavily influenced by implementations from the Vapor and NozeIO frameworks. +This product was heavily influenced by Vapor and NozeIO. * NozeIO * LICENSE (Apache License 2.0) @@ -41,71 +41,36 @@ This product was heavily influenced by implementations from the Vapor and NozeIO --- -This product contains the entire library from Apple's SwiftLog API. +This product contains a a derivation of Georges Menie's crc16 algorithm that was adopted +to Redis coding style by Salvatore Sanfilippo. -* LICENSE (Apache License 2.0): - * https://github.com/apple/swift-log/blob/master/LICENSE.txt -* HOMEPAGE: - * https://github.com/apple/swift-log - ---- - -This product contains the entire library from Apple's SwiftMetrics API. - -* LICENSE (Apache License 2.0): - * https://github.com/apple/swift-metrics/blob/master/LICENSE.txt -* HOMEPAGE: - * https://github.com/apple/swift-metrics - ---- - -This product contains the entire library and derivations of various scripts from Apple's SwiftNIO. - - * LICENSE (Apache License 2.0): - * https://github.com/apple/swift-nio/blob/master/LICENSE.txt + * LICENSE: + /* + * Copyright 2001-2010 Georges Menie (www.menie.org) + * Copyright 2010 Salvatore Sanfilippo (adapted to Redis coding style) + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the University of California, Berkeley nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ * HOMEPAGE: - * https://github.com/apple/swift-nio - ---- - -This product contains a derivation of the Tony Stone's 'process_test_files.rb'. - - * LICENSE (Apache License 2.0): - * https://www.apache.org/licenses/LICENSE-2.0 - * HOMEPAGE: - * https://codegists.com/snippet/ruby/generate_xctest_linux_runnerrb_tonystone_ruby - ---- - -This product contains NodeJS's http-parser. - - * LICENSE (MIT): - * https://github.com/nodejs/http-parser/blob/master/LICENSE-MIT - * HOMEPAGE: - * https://github.com/nodejs/http-parser - ---- - -This product contains "cpp_magic.h" from Thomas Nixon & Jonathan Heathcote's uSHET - - * LICENSE (MIT): - * https://github.com/18sg/uSHET/blob/master/LICENSE - * HOMEPAGE: - * https://github.com/18sg/uSHET - ---- - -This product contains "sha1.c" and "sha1.h" from FreeBSD (Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project) - - * LICENSE (BSD-3): - * https://opensource.org/licenses/BSD-3-Clause - * HOMEPAGE: - * https://github.com/freebsd/freebsd/tree/master/sys/crypto - ---- -This product contains "ifaddrs-android.c" and "ifaddrs-android.h" from dxr.mozilla.org. Webrtc is the owner, but we use edited version from mozilla repo (Copyright (c) 2011, The WebRTC project authors. All rights reserved) - - * LICENSE (BSD): - * https://webrtc.googlesource.com/src/+/master/LICENSE - * HOMEPAGE: - * https://webrtc.googlesource.com/ + * https://redis.io/docs/reference/cluster-spec/#appendix-a-crc16-reference-implementation-in-ansi-c diff --git a/Sources/RediStack/RedisHashSlot.swift b/Sources/RediStack/RedisHashSlot.swift new file mode 100644 index 0000000..1f474bc --- /dev/null +++ b/Sources/RediStack/RedisHashSlot.swift @@ -0,0 +1,222 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the RediStack open source project +// +// Copyright (c) 2023 RediStack project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of RediStack project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +typealias HashSlots = [ClosedRange] + +/// A wrapper around UInt16 that represents a Redis HashSlot. A HashSlot determines which +/// Shard to connect to in a Redis Cluster. +public struct RedisHashSlot: Hashable, Sendable { + /// An unknown `HashSlot` value + public static let unknown = RedisHashSlot(.max) + + /// The max valid `HashSlot` value + public static let max = RedisHashSlot(16383) + /// The min valid `HashSlot` value + public static let min = RedisHashSlot(0) + /// The number of all hash slots + public static let count: Int = 16384 + + private var _raw: UInt16 + + private init(_ value: UInt16) { + precondition(value < 16384 || value == .max) + self._raw = value + } +} + +extension RedisHashSlot: Comparable { + public static func < (lhs: RedisHashSlot, rhs: RedisHashSlot) -> Bool { + lhs.rawValue < rhs.rawValue + } +} + +extension RedisHashSlot: RawRepresentable { + public typealias RawValue = UInt16 + + public init?(rawValue: UInt16) { + guard rawValue < 16384 else { return nil } + self._raw = rawValue + } + + public init?(rawValue: Int) { + guard rawValue >= 0, rawValue < 16384 else { return nil } + self._raw = UInt16(rawValue) + } + + public var rawValue: UInt16 { self._raw } +} + +extension RedisHashSlot: Strideable { + public typealias Stride = Int + + public func distance(to other: RedisHashSlot) -> Int { + Int(other.rawValue - self.rawValue) + } + + public func advanced(by n: Int) -> RedisHashSlot { + // we are fine that this might crash. Same behavior as normal Int and friends. + RedisHashSlot(UInt16(Int(self.rawValue) + n)) + } +} + +extension RedisHashSlot: ExpressibleByIntegerLiteral { + public typealias IntegerLiteralType = UInt16 + + public init(integerLiteral value: UInt16) { + precondition(value < 16384 || value == .max) + self._raw = value + } +} + +extension RedisHashSlot: CustomStringConvertible { + public var description: String { + switch self { + case RedisHashSlot.min...RedisHashSlot.max: + return "\(self.rawValue)" + case RedisHashSlot.unknown: + return "unknown" + default: + fatalError("Invalid value: \(self._raw)") + } + } +} + +extension RedisHashSlot { + /// Creates a slot for a given `key`. A ``HashSlot`` is used to determine which shard to connect to. + /// - Parameter key: The key used in a redis command + public init(key: String) { + // Banging is safe because the modulo ensures we are in range + self.init(rawValue: UInt16(crc16(RedisHashSlot.hashTag(forKey: key)) % 16384))! + } + + /// Computes the range of the key that is used to compute the slot for a given key + /// - Parameter key: The key for your operation + /// - Returns: A substring utf8 view that shall be used in the crc16 computation + static func hashTag(forKey key: String) -> Substring.UTF8View { + let utf8View = key.utf8 + + var firstOpenCurly: String.UTF8View.Index? + var index = utf8View.startIndex + + while index < utf8View.endIndex { + defer { index = utf8View.index(after: index) } + + switch utf8View[index] { + case UInt8(ascii: "{") where firstOpenCurly == nil: + firstOpenCurly = index + case UInt8(ascii: "}"): + guard let firstOpenCurly = firstOpenCurly else { + continue + } + + if firstOpenCurly == utf8View.index(before: index) { + // we had a `{}` combination... this means the complete key shall be used for hashing + return utf8View[...] + } + + return utf8View[(utf8View.index(after: firstOpenCurly))..(_ bytes: Bytes) -> UInt16 where Bytes.Element == UInt8 { + var crc: UInt16 = 0 + for byte in bytes { + crc = (crc &<< 8) ^ crc16tab[(Int(crc &>> 8) ^ Int(byte)) & 0x00FF] + } + return crc +} diff --git a/Tests/RediStackTests/RedisHashSlotTests.swift b/Tests/RediStackTests/RedisHashSlotTests.swift new file mode 100644 index 0000000..5b28cce --- /dev/null +++ b/Tests/RediStackTests/RedisHashSlotTests.swift @@ -0,0 +1,78 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the RediStack open source project +// +// Copyright (c) 2023 RediStack project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of RediStack project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@testable import RediStack +import XCTest + +final class RedisHashSlotTests: XCTestCase { + func testEdgeValues() { + XCTAssertEqual(RedisHashSlot.min.rawValue, 0) + XCTAssertEqual(RedisHashSlot.max.rawValue, UInt16(pow(2.0, 14.0)) - 1) + XCTAssertEqual(RedisHashSlot.unknown.rawValue, UInt16.max) + } + + func testExpressibleByIntegerLiteral() { + let value: RedisHashSlot = 123 + XCTAssertEqual(value.rawValue, 123) + } + + func testStridable() { + let value: RedisHashSlot = 123 + XCTAssertEqual(value.advanced(by: 12), 135) + } + + func testComparable() { + let value: RedisHashSlot = 123 + XCTAssertGreaterThan(value.advanced(by: 1), value) + XCTAssertLessThan(value.advanced(by: -1), value) + } + + func testCRC16() { + XCTAssertEqual(crc16("123456789".utf8), 0x31C3) + + // test cases generated here: https://crccalc.com + XCTAssertEqual(crc16("Peter".utf8), 0x5E67) + XCTAssertEqual(crc16("Fabian".utf8), 0x504F) + XCTAssertEqual(crc16("Inverness".utf8), 0x7619) + XCTAssertEqual(crc16("Redis is awesome".utf8), 0x345C) + XCTAssertEqual(crc16([0xFF, 0xFF, 0x00, 0x00]), 0x84C0) + XCTAssertEqual(crc16([0x00, 0x00]), 0x0000) + } + + func testHashTagComputation() { + XCTAssert(RedisHashSlot.hashTag(forKey: "{user1000}.following").elementsEqual(RedisHashSlot.hashTag(forKey: "{user1000}.followers"))) + XCTAssert(RedisHashSlot.hashTag(forKey: "{user1000}.following").elementsEqual("user1000".utf8)) + XCTAssert(RedisHashSlot.hashTag(forKey: "{user1000}.followers").elementsEqual("user1000".utf8)) + + XCTAssert(RedisHashSlot.hashTag(forKey: "foo{}{bar}").elementsEqual("foo{}{bar}".utf8)) + XCTAssert(RedisHashSlot.hashTag(forKey: "foo{{bar}}zap").elementsEqual("{bar".utf8)) + XCTAssert(RedisHashSlot.hashTag(forKey: "foo{bar}{zap}").elementsEqual("bar".utf8)) + XCTAssert(RedisHashSlot.hashTag(forKey: "{}foo{bar}{zap}").elementsEqual("{}foo{bar}{zap}".utf8)) + XCTAssert(RedisHashSlot.hashTag(forKey: "foo").elementsEqual("foo".utf8)) + XCTAssert(RedisHashSlot.hashTag(forKey: "foo}").elementsEqual("foo}".utf8)) + XCTAssert(RedisHashSlot.hashTag(forKey: "{foo}").elementsEqual("foo".utf8)) + XCTAssert(RedisHashSlot.hashTag(forKey: "bar{foo}").elementsEqual("foo".utf8)) + XCTAssert(RedisHashSlot.hashTag(forKey: "bar{}").elementsEqual("bar{}".utf8)) + XCTAssert(RedisHashSlot.hashTag(forKey: "{}").elementsEqual("{}".utf8)) + XCTAssert(RedisHashSlot.hashTag(forKey: "{}bar").elementsEqual("{}bar".utf8)) + } + + func testDescription() { + XCTAssertEqual(String(describing: RedisHashSlot.min), "0") + XCTAssertEqual(String(describing: RedisHashSlot.max), "16383") + XCTAssertEqual(String(describing: RedisHashSlot.unknown), "unknown") + XCTAssertEqual(String(describing: RedisHashSlot(rawValue: 3000)!), "3000") + XCTAssertEqual(String(describing: RedisHashSlot(rawValue: 20)!), "20") + } +}