Merge pull request #1340 from shogo4405/feature/encoding_video_for_live_streaming

Encoding video for live streaming.

Sources/Codec/VTSessionMode.swift

@@ -15,7 +15,7 @@ enum VTSessionMode {
                 height: Int32(videoCodec.settings.videoSize.height),
                 codecType: videoCodec.settings.format.codecType,
                 encoderSpecification: nil,
-                imageBufferAttributes: videoCodec.attributes as CFDictionary?,
+                imageBufferAttributes: videoCodec.imageBufferAttributes(.compression) as CFDictionary?,
                 compressedDataAllocator: nil,
                 outputCallback: nil,
                 refcon: nil,
@@ -42,15 +42,12 @@ enum VTSessionMode {
                 videoCodec.delegate?.videoCodec(videoCodec, errorOccurred: .failedToCreate(status: kVTParameterErr))
                 return nil
             }
-            var attributes = videoCodec.attributes
-            attributes?.removeValue(forKey: kCVPixelBufferWidthKey)
-            attributes?.removeValue(forKey: kCVPixelBufferHeightKey)
             var session: VTDecompressionSession?
             let status = VTDecompressionSessionCreate(
                 allocator: kCFAllocatorDefault,
                 formatDescription: formatDescription,
                 decoderSpecification: nil,
-                imageBufferAttributes: attributes as CFDictionary?,
+                imageBufferAttributes: videoCodec.imageBufferAttributes(.decompression) as CFDictionary?,
                 outputCallback: nil,
                 decompressionSessionOut: &session
             )

Sources/Codec/VideoCodec.swift

@@ -45,18 +45,6 @@ final class VideoCodec<T: VideoCodecDelegate> {
     /// The running value indicating whether the VideoCodec is running.
     private(set) var isRunning: Atomic<Bool> = .init(false)
     var needsSync: Atomic<Bool> = .init(true)
-    var attributes: [NSString: AnyObject]? {
-        guard kVideoCodec_defaultAttributes != nil else {
-            return nil
-        }
-        var attributes: [NSString: AnyObject] = [:]
-        for (key, value) in kVideoCodec_defaultAttributes ?? [:] {
-            attributes[key] = value
-        }
-        attributes[kCVPixelBufferWidthKey] = NSNumber(value: settings.videoSize.width)
-        attributes[kCVPixelBufferHeightKey] = NSNumber(value: settings.videoSize.height)
-        return attributes
-    }
     var passthrough = true
     var frameInterval = kVideoCodec_defaultFrameInterval
     var expectedFrameRate = IOMixer.defaultFrameRate
@@ -166,6 +154,23 @@ final class VideoCodec<T: VideoCodecDelegate> {
         }
     }
 
+    func imageBufferAttributes(_ mode: VTSessionMode) -> [NSString: AnyObject]? {
+        switch mode {
+        case .compression:
+            var attributes: [NSString: AnyObject] = [:]
+            if let inputFormat {
+                // Specify the pixel format of the uncompressed video.
+                attributes[kCVPixelBufferPixelFormatTypeKey] = inputFormat._mediaSubType as CFNumber
+            }
+            return attributes.isEmpty ? nil : attributes
+        case .decompression:
+            return [
+                kCVPixelBufferIOSurfacePropertiesKey: NSDictionary(),
+                kCVPixelBufferMetalCompatibilityKey: kCFBooleanTrue
+            ]
+        }
+    }
+
     private func willDropFrame(_ presentationTimeStamp: CMTime) -> Bool {
         guard startedAt <= presentationTimeStamp else {
             return true

Sources/Codec/VideoCodecSettings.swift

@@ -86,8 +86,6 @@ public struct VideoCodecSettings: Codable {
     public var videoSize: CGSize
     /// Specifies the bitrate.
     public var bitRate: Int
-    /// Specifies the video frame interval.
-    public var frameInterval: Double
     /// Specifies the H264 profileLevel.
     public var profileLevel: String {
         didSet {
@@ -106,8 +104,12 @@ public struct VideoCodecSettings: Codable {
     public var maxKeyFrameIntervalDuration: Int32
     /// Specifies the allowFrameRecording.
     public var allowFrameReordering: Bool? // swiftlint:disable:this discouraged_optional_boolean
+    /// Specifies the dataRateLimits
+    public var dataRateLimits: [Double]?
     /// Specifies the HardwareEncoder is enabled(TRUE), or not(FALSE) for macOS.
     public var isHardwareEncoderEnabled: Bool
+    /// Specifies the video frame interval.
+    public var frameInterval: Double = 0.0
 
     var format: Format = .h264
 
@@ -115,22 +117,22 @@ public struct VideoCodecSettings: Codable {
     public init(
         videoSize: CGSize = .init(width: 854, height: 480),
         bitRate: Int = 640 * 1000,
-        frameInterval: Double = 0.0,
         profileLevel: String = kVTProfileLevel_H264_Baseline_3_1 as String,
         scalingMode: ScalingMode = .trim,
         bitRateMode: BitRateMode = .average,
         maxKeyFrameIntervalDuration: Int32 = 2,
         allowFrameReordering: Bool? = nil, // swiftlint:disable:this discouraged_optional_boolean,
+        dataRateLimits: [Double]? = [0.0, 0.0],
         isHardwareEncoderEnabled: Bool = true
     ) {
         self.videoSize = videoSize
         self.bitRate = bitRate
-        self.frameInterval = frameInterval
         self.profileLevel = profileLevel
         self.scalingMode = scalingMode
         self.bitRateMode = bitRateMode
         self.maxKeyFrameIntervalDuration = maxKeyFrameIntervalDuration
         self.allowFrameReordering = allowFrameReordering
+        self.dataRateLimits = dataRateLimits
         self.isHardwareEncoderEnabled = isHardwareEncoderEnabled
         if profileLevel.contains("HEVC") {
             self.format = .hevc
@@ -144,6 +146,7 @@ public struct VideoCodecSettings: Codable {
                     allowFrameReordering == rhs.allowFrameReordering &&
                     bitRateMode == rhs.bitRateMode &&
                     profileLevel == rhs.profileLevel &&
+                    dataRateLimits == rhs.dataRateLimits &&
                     isHardwareEncoderEnabled == rhs.isHardwareEncoderEnabled
         )
     }
@@ -161,6 +164,7 @@ public struct VideoCodecSettings: Codable {
         }
     }
 
+    // https://developer.apple.com/documentation/videotoolbox/encoding_video_for_live_streaming
     func options<T>(_ codec: VideoCodec<T>) -> Set<VTSessionOption> {
         let isBaseline = profileLevel.contains("Baseline")
         var options = Set<VTSessionOption>([
@@ -175,6 +179,14 @@ public struct VideoCodecSettings: Codable {
                 "ScalingMode": scalingMode.rawValue
             ] as NSObject)
         ])
+        if bitRateMode == .average {
+            if let dataRateLimits, dataRateLimits.count == 2 {
+                var limits: [Double] = []
+                limits[0] = dataRateLimits[0] == 0 ? Double(bitRate) / 8 * 1.5 : dataRateLimits[0]
+                limits[1] = dataRateLimits[1] == 0 ? Double(1.0) : dataRateLimits[1]
+                options.insert(.init(key: .dataRateLimits, value: limits as NSArray))
+            }
+        }
         #if os(macOS)
         if isHardwareEncoderEnabled {
             options.insert(.init(key: .encoderID, value: format.encoderID))
@@ -182,7 +194,7 @@ public struct VideoCodecSettings: Codable {
             options.insert(.init(key: .requireHardwareAcceleratedVideoEncoder, value: kCFBooleanTrue))
         }
         #endif
-        if !isBaseline {
+        if !isBaseline && profileLevel.contains("H264") {
             options.insert(.init(key: .H264EntropyMode, value: kVTH264EntropyMode_CABAC))
         }
         return options