From 7b658fbe37f97df7b7f37cd9b779d14b74607222 Mon Sep 17 00:00:00 2001 From: shogo4405 Date: Sun, 4 Dec 2022 17:39:03 +0900 Subject: [PATCH] Add MultiCamCaptureSetting#cornerRadius option. --- Examples/iOS/LiveViewController.swift | 5 +- HaishinKit.xcodeproj/project.pbxproj | 16 ++-- Platforms/iOS/IOVideoUnit+Extension.swift | 42 ----------- Platforms/iOS/NetStream+Extension.swift | 6 -- Platforms/iOS/ScreenCaptureSession.swift | 13 ---- .../macOS/IOVideoUnit+Extension-macOS.swift | 24 ------ .../macOS/NetStream+Extension-macOS.swift | 14 ---- .../Extension/CMSampleBuffer+Extension.swift | 11 +-- .../Extension/vImage_Buffer+Extension.swift | 31 ++++++++ Sources/Media/CaptureSessionConvertible.swift | 15 ++++ Sources/Media/IOVideoUnit.swift | 73 +++++++++++++++++-- Sources/Media/MultiCamCaptureSetting.swift | 8 +- Sources/Net/NetStream.swift | 14 ++++ 13 files changed, 151 insertions(+), 121 deletions(-) delete mode 100644 Platforms/macOS/IOVideoUnit+Extension-macOS.swift delete mode 100644 Platforms/macOS/NetStream+Extension-macOS.swift create mode 100644 Sources/Media/CaptureSessionConvertible.swift diff --git a/Examples/iOS/LiveViewController.swift b/Examples/iOS/LiveViewController.swift index bf8085f9..64dd2254 100644 --- a/Examples/iOS/LiveViewController.swift +++ b/Examples/iOS/LiveViewController.swift @@ -109,7 +109,10 @@ final class LiveViewController: UIViewController { currentFrame.origin.x += deltaX currentFrame.origin.y += deltaY pipIntentView.frame = currentFrame - rtmpStream.multiCamCaptureSettings = MultiCamCaptureSetting(regionOfInterest: currentFrame) + rtmpStream.multiCamCaptureSettings = MultiCamCaptureSetting( + cornerRadius: 16.0, + regionOfInterest: currentFrame + ) } } diff --git a/HaishinKit.xcodeproj/project.pbxproj b/HaishinKit.xcodeproj/project.pbxproj index 1df9e3f9..a9d8413a 100644 --- a/HaishinKit.xcodeproj/project.pbxproj +++ b/HaishinKit.xcodeproj/project.pbxproj @@ -131,7 +131,6 @@ 297E69132324E38800D418AB /* AudioCodecFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = 297E69112324E38800D418AB /* AudioCodecFormat.swift */; }; 297E69142324E38800D418AB /* AudioCodecFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = 297E69112324E38800D418AB /* AudioCodecFormat.swift */; }; 298BCF331DD4C44A007FF86A /* AnyUtil.swift in Sources */ = {isa = PBXBuildFile; fileRef = 298BCF321DD4C44A007FF86A /* AnyUtil.swift */; }; - 2992D1541ED04A2C008D9DC1 /* IOVideoUnit+Extension-macOS.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2992D1521ED04A1B008D9DC1 /* IOVideoUnit+Extension-macOS.swift */; }; 2999C3752071138F00892E55 /* MTHKView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2999C3742071138F00892E55 /* MTHKView.swift */; }; 299B13271D3B751400A1E8F5 /* HKView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 299B13261D3B751400A1E8F5 /* HKView.swift */; }; 29AF3FCF1D7C744C00E41212 /* NetStream.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29AF3FCE1D7C744C00E41212 /* NetStream.swift */; }; @@ -213,7 +212,6 @@ 29D0E3681DD4CE3700863B3B /* AnyUtil.swift in Sources */ = {isa = PBXBuildFile; fileRef = 298BCF321DD4C44A007FF86A /* AnyUtil.swift */; }; 29D3D4CF1ED04C4C00DD4AA6 /* IOVideoUnit+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29D3D4CE1ED04C4C00DD4AA6 /* IOVideoUnit+Extension.swift */; }; 29D3D4D11ED04D1200DD4AA6 /* NetStream+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29D3D4D01ED04D1200DD4AA6 /* NetStream+Extension.swift */; }; - 29D3D4D31ED04D9600DD4AA6 /* NetStream+Extension-macOS.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29D3D4D21ED04D9600DD4AA6 /* NetStream+Extension-macOS.swift */; }; 29D3D4D61ED04EAE00DD4AA6 /* DeviceUtil+Extenstion.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29D3D4D41ED04E7100DD4AA6 /* DeviceUtil+Extenstion.swift */; }; 29D3D4EA1ED0519400DD4AA6 /* HaishinKit.h in Headers */ = {isa = PBXBuildFile; fileRef = 29D3D4E61ED0518900DD4AA6 /* HaishinKit.h */; settings = {ATTRIBUTES = (Public, ); }; }; 29DC17B321D0CC0600E26CED /* Atomic.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29DC17B221D0CC0600E26CED /* Atomic.swift */; }; @@ -405,6 +403,9 @@ BC9F9C7826F8C16600B01ED0 /* Choreographer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */; }; BC9F9C7926F8C16600B01ED0 /* Choreographer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */; }; BC9F9C7A26F8C16600B01ED0 /* Choreographer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */; }; + BCA2252C293CC5B600DD7CB2 /* CaptureSessionConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCA2252B293CC5B600DD7CB2 /* CaptureSessionConvertible.swift */; }; + BCA2252D293CC5B600DD7CB2 /* CaptureSessionConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCA2252B293CC5B600DD7CB2 /* CaptureSessionConvertible.swift */; }; + BCA2252E293CC5B600DD7CB2 /* CaptureSessionConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCA2252B293CC5B600DD7CB2 /* CaptureSessionConvertible.swift */; }; BCA97B86263AC0F30027213C /* MP4BoxConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCA97B85263AC0F30027213C /* MP4BoxConvertible.swift */; }; BCA97B87263AC0F30027213C /* MP4BoxConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCA97B85263AC0F30027213C /* MP4BoxConvertible.swift */; }; BCA97B88263AC0F30027213C /* MP4BoxConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCA97B85263AC0F30027213C /* MP4BoxConvertible.swift */; }; @@ -760,7 +761,6 @@ 297E69112324E38800D418AB /* AudioCodecFormat.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioCodecFormat.swift; sourceTree = ""; }; 2981E1301D646E3F00E8F7CA /* Cartfile */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = Cartfile; sourceTree = ""; }; 298BCF321DD4C44A007FF86A /* AnyUtil.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AnyUtil.swift; sourceTree = ""; }; - 2992D1521ED04A1B008D9DC1 /* IOVideoUnit+Extension-macOS.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "IOVideoUnit+Extension-macOS.swift"; sourceTree = ""; }; 2997BDD31D50D31B000AF900 /* README.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = ""; }; 2999C3742071138F00892E55 /* MTHKView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MTHKView.swift; sourceTree = ""; }; 299B131C1D35272D00A1E8F5 /* ScreenCaptureSession.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ScreenCaptureSession.swift; sourceTree = ""; }; @@ -814,7 +814,6 @@ 29C932941CD76FD300283FC5 /* Example macOS.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Example macOS.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 29D3D4CE1ED04C4C00DD4AA6 /* IOVideoUnit+Extension.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "IOVideoUnit+Extension.swift"; sourceTree = ""; }; 29D3D4D01ED04D1200DD4AA6 /* NetStream+Extension.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "NetStream+Extension.swift"; sourceTree = ""; }; - 29D3D4D21ED04D9600DD4AA6 /* NetStream+Extension-macOS.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "NetStream+Extension-macOS.swift"; sourceTree = ""; }; 29D3D4D41ED04E7100DD4AA6 /* DeviceUtil+Extenstion.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "DeviceUtil+Extenstion.swift"; sourceTree = ""; }; 29D3D4DC1ED0509900DD4AA6 /* HaishinKit.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = HaishinKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 29D3D4E51ED0518900DD4AA6 /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; @@ -870,6 +869,7 @@ BC94E52C264146120094C169 /* MP4ReaderConvertible.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MP4ReaderConvertible.swift; sourceTree = ""; }; BC9CFA9223BDE8B700917EEF /* NetStreamDrawable.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NetStreamDrawable.swift; sourceTree = ""; }; BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Choreographer.swift; sourceTree = ""; }; + BCA2252B293CC5B600DD7CB2 /* CaptureSessionConvertible.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CaptureSessionConvertible.swift; sourceTree = ""; }; BCA97B85263AC0F30027213C /* MP4BoxConvertible.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MP4BoxConvertible.swift; sourceTree = ""; }; BCA97B89263AC1830027213C /* MP4Box.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MP4Box.swift; sourceTree = ""; }; BCA97B8D263AC49B0027213C /* MP4MediaHeaderBox.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MP4MediaHeaderBox.swift; sourceTree = ""; }; @@ -1262,8 +1262,6 @@ 296543591D62FE6200734698 /* HaishinKit.h */, 2965435A1D62FE6200734698 /* HKView-macOS.swift */, 296543581D62FE6200734698 /* Info.plist */, - 2992D1521ED04A1B008D9DC1 /* IOVideoUnit+Extension-macOS.swift */, - 29D3D4D21ED04D9600DD4AA6 /* NetStream+Extension-macOS.swift */, ); path = macOS; sourceTree = ""; @@ -1369,6 +1367,7 @@ isa = PBXGroup; children = ( 2941746A22D069B300A2944F /* AudioEffect.swift */, + BCA2252B293CC5B600DD7CB2 /* CaptureSessionConvertible.swift */, BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */, 29B876891CD70AFE00FC07DA /* IOAudioUnit.swift */, BC1102492925147300D48035 /* IOCaptureUnit.swift */, @@ -2136,6 +2135,7 @@ BCC1A7132647F28F00661156 /* SLConfigDescriptor.swift in Sources */, 29DF20622312A3DD004057C3 /* RTMPNWSocket.swift in Sources */, 29B876BD1CD70B3900FC07DA /* CRC32.swift in Sources */, + BCA2252C293CC5B600DD7CB2 /* CaptureSessionConvertible.swift in Sources */, BC4914A628DDD367009E2DF6 /* VTSessionOption.swift in Sources */, BC4914B228DDFE31009E2DF6 /* VTSessionOptionKey.swift in Sources */, 29EA87E61E79A2780043A5F8 /* CMAudioFormatDescription+Extension.swift in Sources */, @@ -2253,6 +2253,7 @@ BCB977402621812800C9A649 /* AVCFormatStream.swift in Sources */, 29B876EE1CD70D5900FC07DA /* VideoCodec.swift in Sources */, BCA97BF1263C31020027213C /* MP4SampleDescriptionBox.swift in Sources */, + BCA2252D293CC5B600DD7CB2 /* CaptureSessionConvertible.swift in Sources */, 29EA87EB1E79A3B70043A5F8 /* CMBlockBuffer+Extension.swift in Sources */, 29B876F01CD70D5900FC07DA /* Constants.swift in Sources */, 29EA87D91E79A0090043A5F8 /* URL+Extension.swift in Sources */, @@ -2272,7 +2273,6 @@ BC4C9EB023F2E736004A14F2 /* AudioStreamBasicDescription+Extension.swift in Sources */, 2942A4F921A9418A004E1BEE /* Running.swift in Sources */, BC110254292DD6E900D48035 /* vImage_Buffer+Extension.swift in Sources */, - 29D3D4D31ED04D9600DD4AA6 /* NetStream+Extension-macOS.swift in Sources */, 29EA87D61E799F6A0043A5F8 /* Mirror+Extension.swift in Sources */, BC7A23F525171C8F0089F77C /* MTHKView.swift in Sources */, 297E69132324E38800D418AB /* AudioCodecFormat.swift in Sources */, @@ -2322,7 +2322,6 @@ BCC1A6E826447CCA00661156 /* MP4MovieHeaderBox.swift in Sources */, 29B877051CD70D5A00FC07DA /* IOMixer.swift in Sources */, 294B2D3323785E3800CE7BDC /* CircularBuffer.swift in Sources */, - 2992D1541ED04A2C008D9DC1 /* IOVideoUnit+Extension-macOS.swift in Sources */, 2976A47F1D48FD6900B53EF2 /* IORecorder.swift in Sources */, BCA97BF5263C390E0027213C /* CustomXmlStringConvertible.swift in Sources */, BC44A1AA23D31E92002D4297 /* AudioCodecBuffer.swift in Sources */, @@ -2479,6 +2478,7 @@ 29EB3E131ED05887001CAE8B /* SoundTransform.swift in Sources */, 29EB3DF81ED05799001CAE8B /* Mirror+Extension.swift in Sources */, BCA97C1A263DA5070027213C /* MP4AudioSampleEntry.swift in Sources */, + BCA2252E293CC5B600DD7CB2 /* CaptureSessionConvertible.swift in Sources */, BC83A4752403D83B006BDE06 /* VTCompressionSession+Extension.swift in Sources */, BC9CFA9523BDE8B700917EEF /* NetStreamDrawable.swift in Sources */, BCC1A6E926447CCA00661156 /* MP4MovieHeaderBox.swift in Sources */, diff --git a/Platforms/iOS/IOVideoUnit+Extension.swift b/Platforms/iOS/IOVideoUnit+Extension.swift index 1e41b092..8a159782 100644 --- a/Platforms/iOS/IOVideoUnit+Extension.swift +++ b/Platforms/iOS/IOVideoUnit+Extension.swift @@ -27,48 +27,6 @@ extension IOVideoUnit { logger.error("while locking device for ramp:", error) } } - - func attachScreen(_ screen: CaptureSessionConvertible?, useScreenSize: Bool = true) { - guard let screen = screen else { - self.screen?.stopRunning() - self.screen = nil - return - } - capture = nil - if useScreenSize { - codec.width = screen.attributes["Width"] as! Int32 - codec.height = screen.attributes["Height"] as! Int32 - } - self.screen = screen - } -} - -extension IOVideoUnit: CaptureSessionDelegate { - // MARK: CaptureSessionDelegate - func session(_ session: CaptureSessionConvertible, didSet size: CGSize) { - lockQueue.async { - self.codec.width = Int32(size.width) - self.codec.height = Int32(size.height) - } - } - - func session(_ session: CaptureSessionConvertible, didOutput pixelBuffer: CVPixelBuffer, presentationTime: CMTime) { - if !effects.isEmpty { - // usually the context comes from HKView or MTLHKView - // but if you have not attached a view then the context is nil - if context == nil { - logger.info("no ci context, creating one to render effect") - context = CIContext() - } - context?.render(effect(pixelBuffer, info: nil), to: pixelBuffer) - } - codec.inputBuffer( - pixelBuffer, - presentationTimeStamp: presentationTime, - duration: CMTime.invalid - ) - mixer?.recorder.appendPixelBuffer(pixelBuffer, withPresentationTime: presentationTime) - } } #endif diff --git a/Platforms/iOS/NetStream+Extension.swift b/Platforms/iOS/NetStream+Extension.swift index 0442be88..5413f356 100644 --- a/Platforms/iOS/NetStream+Extension.swift +++ b/Platforms/iOS/NetStream+Extension.swift @@ -13,12 +13,6 @@ extension NetStream { } } - public func attachScreen(_ screen: CaptureSessionConvertible?, useScreenSize: Bool = true) { - lockQueue.async { - self.mixer.videoIO.attachScreen(screen, useScreenSize: useScreenSize) - } - } - public var zoomFactor: CGFloat { self.mixer.videoIO.zoomFactor } diff --git a/Platforms/iOS/ScreenCaptureSession.swift b/Platforms/iOS/ScreenCaptureSession.swift index 9c797595..59b89fad 100644 --- a/Platforms/iOS/ScreenCaptureSession.swift +++ b/Platforms/iOS/ScreenCaptureSession.swift @@ -4,25 +4,12 @@ import AVFoundation import CoreImage import UIKit -/// The interface a capture session uses to inform its delegate. -public protocol CaptureSessionDelegate: AnyObject { - /// Tells the receiver to set a size. - func session(_ session: CaptureSessionConvertible, didSet size: CGSize) - /// Tells the receiver to output a pixel buffer. - func session(_ session: CaptureSessionConvertible, didOutput pixelBuffer: CVPixelBuffer, presentationTime: CMTime) -} - extension CGRect { init(size: CGSize) { self.init(origin: .zero, size: size) } } -public protocol CaptureSessionConvertible: Running { - var attributes: [NSString: NSObject] { get } - var delegate: CaptureSessionDelegate? { get set } -} - // MARK: - open class ScreenCaptureSession: NSObject, CaptureSessionConvertible { static let defaultFrameInterval: Int = 2 diff --git a/Platforms/macOS/IOVideoUnit+Extension-macOS.swift b/Platforms/macOS/IOVideoUnit+Extension-macOS.swift deleted file mode 100644 index 2db64a54..00000000 --- a/Platforms/macOS/IOVideoUnit+Extension-macOS.swift +++ /dev/null @@ -1,24 +0,0 @@ -#if os(macOS) - -import AVFoundation - -extension IOVideoUnit { - func attachScreen(_ screen: AVCaptureScreenInput?) { - mixer?.session.beginConfiguration() - defer { - mixer?.session.commitConfiguration() - } - guard let screen else { - capture = nil - return - } - let output = AVCaptureVideoDataOutput() - output.alwaysDiscardsLateVideoFrames = true - output.videoSettings = videoSettings as? [String: Any] - capture = IOCaptureUnit(input: screen, output: output, connection: nil) - capture?.attach(mixer?.session) - capture?.output.setSampleBufferDelegate(self, queue: lockQueue) - } -} - -#endif diff --git a/Platforms/macOS/NetStream+Extension-macOS.swift b/Platforms/macOS/NetStream+Extension-macOS.swift deleted file mode 100644 index 1ba12223..00000000 --- a/Platforms/macOS/NetStream+Extension-macOS.swift +++ /dev/null @@ -1,14 +0,0 @@ -#if os(macOS) - -import AVFoundation -import Foundation - -extension NetStream { - public func attachScreen(_ screen: AVCaptureScreenInput?) { - lockQueue.async { - self.mixer.videoIO.attachScreen(screen) - } - } -} - -#endif diff --git a/Sources/Extension/CMSampleBuffer+Extension.swift b/Sources/Extension/CMSampleBuffer+Extension.swift index 4165a98d..5a8a27fd 100644 --- a/Sources/Extension/CMSampleBuffer+Extension.swift +++ b/Sources/Extension/CMSampleBuffer+Extension.swift @@ -102,7 +102,7 @@ extension CMSampleBuffer { return self } - func over(_ sampleBuffer: CMSampleBuffer?, regionOfInterest roi: CGRect = .zero) -> CMSampleBuffer { + func over(_ sampleBuffer: CMSampleBuffer?, regionOfInterest roi: CGRect = .zero, radius: CGFloat = 0.0) -> CMSampleBuffer { guard var inputImageBuffer = vImage_Buffer(cvPixelBuffer: sampleBuffer?.imageBuffer, format: &CMSampleBuffer.format) else { return self } @@ -118,12 +118,13 @@ extension CMSampleBuffer { let xScale = Float(roi.width) / Float(inputImageBuffer.width) let yScale = Float(roi.height) / Float(inputImageBuffer.height) let scaleFactor = (xScale < yScale) ? xScale : yScale - var scaledInputImageBuffer = inputImageBuffer.scale(scaleFactor ) + var scaledInputImageBuffer = inputImageBuffer.scale(scaleFactor) + scaledInputImageBuffer.cornerRadius(radius) defer { scaledInputImageBuffer.free() } - _ = srcImageBuffer.over(&scaledInputImageBuffer, origin: roi.origin) - _ = srcImageBuffer.copy(to: imageBuffer, format: &CMSampleBuffer.format) + srcImageBuffer.over(&scaledInputImageBuffer, origin: roi.origin) + srcImageBuffer.copy(to: imageBuffer, format: &CMSampleBuffer.format) return self } @@ -179,7 +180,7 @@ extension CMSampleBuffer { vImage_Flags(kvImageLeaveAlphaUnchanged)) == kvImageNoError else { return } - _ = destinationBuffer.copy(to: imageBuffer, format: &CMSampleBuffer.format) + destinationBuffer.copy(to: imageBuffer, format: &CMSampleBuffer.format) } #endif } diff --git a/Sources/Extension/vImage_Buffer+Extension.swift b/Sources/Extension/vImage_Buffer+Extension.swift index c6b20afb..9480a115 100644 --- a/Sources/Extension/vImage_Buffer+Extension.swift +++ b/Sources/Extension/vImage_Buffer+Extension.swift @@ -33,6 +33,7 @@ extension vImage_Buffer { } } + @discardableResult mutating func copy(to cvPixelBuffer: CVPixelBuffer, format: inout vImage_CGImageFormat) -> vImage_Error { let cvImageFormat = vImageCVImageFormat_CreateWithCVPixelBuffer(cvPixelBuffer).takeRetainedValue() vImageCVImageFormat_SetColorSpace(cvImageFormat, CGColorSpaceCreateDeviceRGB()) @@ -45,6 +46,7 @@ extension vImage_Buffer { vImage_Flags(kvImageNoFlags)) } + @discardableResult mutating func scale(_ factor: Float) -> Self { var imageBuffer = vImage_Buffer() guard vImageBuffer_Init( @@ -65,6 +67,35 @@ extension vImage_Buffer { return imageBuffer } + @discardableResult + mutating func cornerRadius(_ radius: CGFloat) -> Self { + guard 0 < radius else { + return self + } + let buffer = data.assumingMemoryBound(to: Pixel_8.self) + for x in 0 ..< Int(width) { + for y in 0 ..< Int(height) { + let index = y * rowBytes + x * 4 + var dx = CGFloat(min(x, Int(width) - x)) + var dy = CGFloat(min(y, Int(height) - y)) + if dx == 0 && dy == 0 { + buffer[index] = 0 + continue + } + if radius < dx || radius < dy { + continue + } + dx = radius - dx + dy = radius - dy + if radius < round(sqrt(dx * dx + dy * dy)) { + buffer[index] = 0 + } + } + } + return self + } + + @discardableResult mutating func over(_ src: inout vImage_Buffer, origin: CGPoint = .zero) -> Self { let start = Int(origin.y) * rowBytes + Int(origin.x) * 4 var destination = vImage_Buffer( diff --git a/Sources/Media/CaptureSessionConvertible.swift b/Sources/Media/CaptureSessionConvertible.swift new file mode 100644 index 00000000..af9d51a4 --- /dev/null +++ b/Sources/Media/CaptureSessionConvertible.swift @@ -0,0 +1,15 @@ +import CoreMedia +import Foundation + +/// The interface a capture session uses to inform its delegate. +public protocol CaptureSessionDelegate: AnyObject { + /// Tells the receiver to set a size. + func session(_ session: CaptureSessionConvertible, didSet size: CGSize) + /// Tells the receiver to output a pixel buffer. + func session(_ session: CaptureSessionConvertible, didOutput pixelBuffer: CVPixelBuffer, presentationTime: CMTime) +} + +public protocol CaptureSessionConvertible: Running { + var attributes: [NSString: NSObject] { get } + var delegate: CaptureSessionDelegate? { get set } +} diff --git a/Sources/Media/IOVideoUnit.swift b/Sources/Media/IOVideoUnit.swift index 5743317c..dfd85142 100644 --- a/Sources/Media/IOVideoUnit.swift +++ b/Sources/Media/IOVideoUnit.swift @@ -240,14 +240,14 @@ final class IOVideoUnit: NSObject, IOUnit { } } - var capture: IOCaptureUnit? { + private(set) var capture: IOCaptureUnit? { didSet { oldValue?.output.setSampleBufferDelegate(nil, queue: nil) oldValue?.detachSession(mixer?.session) } } - var multiCamCapture: IOCaptureUnit? { + private(set) var multiCamCapture: IOCaptureUnit? { didSet { oldValue?.output.setSampleBufferDelegate(nil, queue: nil) oldValue?.detachSession(mixer?.session) @@ -257,8 +257,7 @@ final class IOVideoUnit: NSObject, IOUnit { var multiCamCaptureSettings: MultiCamCaptureSetting = .default #endif - #if os(iOS) - var screen: CaptureSessionConvertible? { + private(set) var screen: CaptureSessionConvertible? { didSet { if let oldValue = oldValue { oldValue.delegate = nil @@ -268,7 +267,6 @@ final class IOVideoUnit: NSObject, IOUnit { } } } - #endif private var pixelBuffer: CVPixelBuffer? private var multiCamSampleBuffer: CMSampleBuffer? @@ -402,6 +400,41 @@ final class IOVideoUnit: NSObject, IOUnit { } #endif + #if os(macOS) + func attachScreen(_ screen: AVCaptureScreenInput?) { + mixer?.session.beginConfiguration() + defer { + mixer?.session.commitConfiguration() + } + guard let screen else { + capture = nil + return + } + let output = AVCaptureVideoDataOutput() + output.alwaysDiscardsLateVideoFrames = true + output.videoSettings = videoSettings as? [String: Any] + capture = IOCaptureUnit(input: screen, output: output, connection: nil) + capture?.attachSession(mixer?.session) + capture?.output.setSampleBufferDelegate(self, queue: lockQueue) + } + #endif + + func attachScreen(_ screen: CaptureSessionConvertible?, useScreenSize: Bool = true) { + guard let screen = screen else { + self.screen?.stopRunning() + self.screen = nil + return + } + #if os(iOS) || os(macOS) + capture = nil + #endif + if useScreenSize { + codec.width = screen.attributes["Width"] as! Int32 + codec.height = screen.attributes["Height"] as! Int32 + } + self.screen = screen + } + @inline(__always) func effect(_ buffer: CVImageBuffer, info: CMSampleBuffer?) -> CIImage { var image = CIImage(cvPixelBuffer: buffer) @@ -515,7 +548,7 @@ extension IOVideoUnit: AVCaptureVideoDataOutputSampleBufferDelegate { sampleBuffer.reflectHorizontal() } #endif - appendSampleBuffer(sampleBuffer.over(multiCamSampleBuffer, regionOfInterest: multiCamCaptureSettings.regionOfInterest)) + appendSampleBuffer(sampleBuffer.over(multiCamSampleBuffer, regionOfInterest: multiCamCaptureSettings.regionOfInterest, radius: multiCamCaptureSettings.cornerRadius)) } else if multiCamCapture?.output == captureOutput { multiCamSampleBuffer = sampleBuffer } @@ -535,3 +568,31 @@ extension IOVideoUnit: VideoCodecDelegate { func videoCodec(_ codec: VideoCodec, errorOccurred error: VideoCodec.Error) { } } + +extension IOVideoUnit: CaptureSessionDelegate { + // MARK: CaptureSessionDelegate + func session(_ session: CaptureSessionConvertible, didSet size: CGSize) { + lockQueue.async { + self.codec.width = Int32(size.width) + self.codec.height = Int32(size.height) + } + } + + func session(_ session: CaptureSessionConvertible, didOutput pixelBuffer: CVPixelBuffer, presentationTime: CMTime) { + if !effects.isEmpty { + // usually the context comes from HKView or MTLHKView + // but if you have not attached a view then the context is nil + if context == nil { + logger.info("no ci context, creating one to render effect") + context = CIContext() + } + context?.render(effect(pixelBuffer, info: nil), to: pixelBuffer) + } + codec.inputBuffer( + pixelBuffer, + presentationTimeStamp: presentationTime, + duration: CMTime.invalid + ) + mixer?.recorder.appendPixelBuffer(pixelBuffer, withPresentationTime: presentationTime) + } +} diff --git a/Sources/Media/MultiCamCaptureSetting.swift b/Sources/Media/MultiCamCaptureSetting.swift index e8bccbb2..bd787eff 100644 --- a/Sources/Media/MultiCamCaptureSetting.swift +++ b/Sources/Media/MultiCamCaptureSetting.swift @@ -5,17 +5,21 @@ import Foundation /// The MultiCamCaptureSetting represents the pip capture settings for the video capture. public struct MultiCamCaptureSetting { public static let `default` = MultiCamCaptureSetting( + cornerRadius: 16.0, regionOfInterest: .init( origin: CGPoint(x: 16, y: 16), size: .init(width: 160, height: 160) ) ) + /// The cornerRadius of the picture in picture image. + public let cornerRadius: CGFloat /// The region of the picture in picture image. public let regionOfInterest: CGRect - /// Create a new PiPCaptureSetting. - public init(regionOfInterest: CGRect) { + /// Create a new MultiCamCaptureSetting. + public init(cornerRadius: CGFloat, regionOfInterest: CGRect) { + self.cornerRadius = cornerRadius self.regionOfInterest = regionOfInterest } } diff --git a/Sources/Net/NetStream.swift b/Sources/Net/NetStream.swift index a4133e11..32811f4d 100644 --- a/Sources/Net/NetStream.swift +++ b/Sources/Net/NetStream.swift @@ -174,7 +174,21 @@ open class NetStream: NSObject { mixer.videoIO.focusPointOfInterest = focus mixer.videoIO.exposurePointOfInterest = exposure } + + #if os(macOS) + public func attachScreen(_ screen: AVCaptureScreenInput?) { + lockQueue.async { + self.mixer.videoIO.attachScreen(screen) + } + } #endif + #endif + + open func attachScreen(_ screen: CaptureSessionConvertible?, useScreenSize: Bool = true) { + lockQueue.async { + self.mixer.videoIO.attachScreen(screen, useScreenSize: useScreenSize) + } + } /// Append a CMSampleBuffer?. /// - Warning: This method can't use attachCamera or attachAudio method at the same time.