From 26e1606f1099a19119adf201a6794de8965782db Mon Sep 17 00:00:00 2001 From: shogo4405 Date: Sat, 3 Dec 2022 18:12:15 +0900 Subject: [PATCH] Support multi camera capture on macOS. --- Examples/iOS/LiveViewController.swift | 6 +-- HaishinKit.xcodeproj/project.pbxproj | 16 +++---- Sources/Media/AVAudioIOUnit.swift | 4 +- Sources/Media/AVMixer.swift | 21 +++++++++ Sources/Media/AVVideoIOUnit.swift | 47 ++++++++----------- ...ing.swift => MultiCamCaptureSetting.swift} | 10 ++-- Sources/Net/NetStream.swift | 15 +++--- 7 files changed, 64 insertions(+), 55 deletions(-) rename Sources/Media/{PiPCaptureSetting.swift => MultiCamCaptureSetting.swift} (63%) diff --git a/Examples/iOS/LiveViewController.swift b/Examples/iOS/LiveViewController.swift index 4020dd85..c09850ef 100644 --- a/Examples/iOS/LiveViewController.swift +++ b/Examples/iOS/LiveViewController.swift @@ -32,7 +32,7 @@ final class LiveViewController: UIViewController { pipIntentView.layer.borderWidth = 1.0 pipIntentView.layer.borderColor = UIColor.white.cgColor - pipIntentView.bounds = PiPCaptureSetting.default.regionOfInterest + pipIntentView.bounds = MultiCamCaptureSetting.default.regionOfInterest pipIntentView.isUserInteractionEnabled = true view.addSubview(pipIntentView) @@ -72,7 +72,7 @@ final class LiveViewController: UIViewController { } if #available(iOS 13.0, *) { let front = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .front) - rtmpStream.attachPiPCamera(front) + rtmpStream.attachMultiCamera(front) } rtmpStream.addObserver(self, forKeyPath: "currentFPS", options: .new, context: nil) lfView?.attachStream(rtmpStream) @@ -109,7 +109,7 @@ final class LiveViewController: UIViewController { currentFrame.origin.x += deltaX currentFrame.origin.y += deltaY pipIntentView.frame = currentFrame - rtmpStream.pipCaptureSettings = PiPCaptureSetting(regionOfInterest: currentFrame) + rtmpStream.multiCamCaptureSettings = MultiCamCaptureSetting(regionOfInterest: currentFrame) } } diff --git a/HaishinKit.xcodeproj/project.pbxproj b/HaishinKit.xcodeproj/project.pbxproj index 65f0760c..ea598a58 100644 --- a/HaishinKit.xcodeproj/project.pbxproj +++ b/HaishinKit.xcodeproj/project.pbxproj @@ -330,9 +330,9 @@ BC110253292DD6E900D48035 /* vImage_Buffer+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110252292DD6E900D48035 /* vImage_Buffer+Extension.swift */; }; BC110254292DD6E900D48035 /* vImage_Buffer+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110252292DD6E900D48035 /* vImage_Buffer+Extension.swift */; }; BC110255292DD6E900D48035 /* vImage_Buffer+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110252292DD6E900D48035 /* vImage_Buffer+Extension.swift */; }; - BC110257292E661E00D48035 /* PiPCaptureSetting.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110256292E661E00D48035 /* PiPCaptureSetting.swift */; }; - BC110258292E661E00D48035 /* PiPCaptureSetting.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110256292E661E00D48035 /* PiPCaptureSetting.swift */; }; - BC110259292E661E00D48035 /* PiPCaptureSetting.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110256292E661E00D48035 /* PiPCaptureSetting.swift */; }; + BC110257292E661E00D48035 /* MultiCamCaptureSetting.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110256292E661E00D48035 /* MultiCamCaptureSetting.swift */; }; + BC110258292E661E00D48035 /* MultiCamCaptureSetting.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110256292E661E00D48035 /* MultiCamCaptureSetting.swift */; }; + BC110259292E661E00D48035 /* MultiCamCaptureSetting.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110256292E661E00D48035 /* MultiCamCaptureSetting.swift */; }; BC11D94625A1B01000D710BA /* Screencast.appex in Embed Foundation Extensions */ = {isa = PBXBuildFile; fileRef = 2915EC521D85BDF100621092 /* Screencast.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; BC20DF38250377A3007BC608 /* ScreenCaptureSession.swift in Sources */ = {isa = PBXBuildFile; fileRef = 299B131C1D35272D00A1E8F5 /* ScreenCaptureSession.swift */; }; BC34DFF025EBB53F005F975A /* Logboard.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = BC34DFD125EBB12C005F975A /* Logboard.xcframework */; }; @@ -849,7 +849,7 @@ BC11023D2917C35B00D48035 /* CVPixelBufferPool+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CVPixelBufferPool+Extension.swift"; sourceTree = ""; }; BC1102492925147300D48035 /* AVCaptureIOUnit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AVCaptureIOUnit.swift; sourceTree = ""; }; BC110252292DD6E900D48035 /* vImage_Buffer+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "vImage_Buffer+Extension.swift"; sourceTree = ""; }; - BC110256292E661E00D48035 /* PiPCaptureSetting.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PiPCaptureSetting.swift; sourceTree = ""; }; + BC110256292E661E00D48035 /* MultiCamCaptureSetting.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MultiCamCaptureSetting.swift; sourceTree = ""; }; BC34DFD125EBB12C005F975A /* Logboard.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = Logboard.xcframework; path = Carthage/Build/Logboard.xcframework; sourceTree = ""; }; BC34FA0A286CB90A00EFAF27 /* PiPHKView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PiPHKView.swift; sourceTree = ""; }; BC44A1A823D31E92002D4297 /* AudioCodecBuffer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioCodecBuffer.swift; sourceTree = ""; wrapsLines = 1; }; @@ -1378,7 +1378,7 @@ BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */, 2901A4ED1D437170002BBD23 /* MediaLink.swift */, 2999C3742071138F00892E55 /* MTHKView.swift */, - BC110256292E661E00D48035 /* PiPCaptureSetting.swift */, + BC110256292E661E00D48035 /* MultiCamCaptureSetting.swift */, BC34FA0A286CB90A00EFAF27 /* PiPHKView.swift */, 295891001EEB7A8B00CE51E1 /* ScalingMode.swift */, 29B8768D1CD70AFE00FC07DA /* SoundTransform.swift */, @@ -2118,7 +2118,7 @@ 2976A47E1D48C5C700B53EF2 /* AVRecorder.swift in Sources */, BCC1A6C52643F41600661156 /* MP4MovieFragmentBox.Builder.swift in Sources */, BC94E4FE263FE6B80094C169 /* MP4MovieFragmentHeaderBox.swift in Sources */, - BC110257292E661E00D48035 /* PiPCaptureSetting.swift in Sources */, + BC110257292E661E00D48035 /* MultiCamCaptureSetting.swift in Sources */, 29B876B21CD70B2800FC07DA /* RTMPMuxer.swift in Sources */, 2958912E1EEB8F4100CE51E1 /* FLVSoundType.swift in Sources */, BC0D236D26331BAB001DDA0C /* NetSocket.CircularBuffer.swift in Sources */, @@ -2299,7 +2299,7 @@ BC83A4742403D83B006BDE06 /* VTCompressionSession+Extension.swift in Sources */, BCC1A72C264FAC1800661156 /* ElementaryStreamSpecificData.swift in Sources */, 29B876FF1CD70D5A00FC07DA /* NALUnit.swift in Sources */, - BC110258292E661E00D48035 /* PiPCaptureSetting.swift in Sources */, + BC110258292E661E00D48035 /* MultiCamCaptureSetting.swift in Sources */, 29B877001CD70D5A00FC07DA /* PacketizedElementaryStream.swift in Sources */, BCC1A6EC2644850F00661156 /* MP4TrackHeaderBox.swift in Sources */, BC570B4928E9ACC10098A12C /* AVIOUnit.swift in Sources */, @@ -2505,7 +2505,7 @@ BCC1A7112647F25A00661156 /* DecoderConfigDescriptor.swift in Sources */, BCC1A6D126445AF500661156 /* MP4TrackExtendsBox.swift in Sources */, BC4914A828DDD367009E2DF6 /* VTSessionOption.swift in Sources */, - BC110259292E661E00D48035 /* PiPCaptureSetting.swift in Sources */, + BC110259292E661E00D48035 /* MultiCamCaptureSetting.swift in Sources */, BC4914A428DDD33D009E2DF6 /* VTSessionConvertible.swift in Sources */, BCAD0C02263E968400ADFB80 /* MP4FileReader.swift in Sources */, BCA97C00263C599C0027213C /* MP4SyncSampleBox.swift in Sources */, diff --git a/Sources/Media/AVAudioIOUnit.swift b/Sources/Media/AVAudioIOUnit.swift index 3029c8a7..6b0be939 100644 --- a/Sources/Media/AVAudioIOUnit.swift +++ b/Sources/Media/AVAudioIOUnit.swift @@ -40,7 +40,7 @@ final class AVAudioIOUnit: NSObject, AVIOUnit { #if os(iOS) || os(macOS) func attachAudio(_ audio: AVCaptureDevice?, automaticallyConfiguresApplicationAudioSession: Bool) throws { - guard let mixer = mixer else { + guard let mixer else { return } mixer.session.beginConfiguration() @@ -48,7 +48,7 @@ final class AVAudioIOUnit: NSObject, AVIOUnit { mixer.session.commitConfiguration() } codec.invalidate() - guard let audio: AVCaptureDevice = audio else { + guard let audio else { capture = nil return } diff --git a/Sources/Media/AVMixer.swift b/Sources/Media/AVMixer.swift index d854be4d..b3f22124 100644 --- a/Sources/Media/AVMixer.swift +++ b/Sources/Media/AVMixer.swift @@ -74,6 +74,27 @@ public class AVMixer { case passthrough } + var isMultiCamSessionEnabled = false { + didSet { + guard oldValue != isMultiCamSessionEnabled else { + return + } + #if os(iOS) + if #available(iOS 13.0, *) { + if isMultiCamSessionEnabled { + if !(session is AVCaptureMultiCamSession) { + session = AVCaptureMultiCamSession() + } + } else { + if session is AVCaptureMultiCamSession { + session = AVCaptureSession() + } + } + } + #endif + } + } + #if os(iOS) var preferredVideoStabilizationMode: AVCaptureVideoStabilizationMode { get { diff --git a/Sources/Media/AVVideoIOUnit.swift b/Sources/Media/AVVideoIOUnit.swift index dd152898..b8ab8fea 100644 --- a/Sources/Media/AVVideoIOUnit.swift +++ b/Sources/Media/AVVideoIOUnit.swift @@ -249,6 +249,8 @@ final class AVVideoIOUnit: NSObject, AVIOUnit { oldValue?.detach(mixer?.session) } } + + var multiCamCaptureSettings: MultiCamCaptureSetting = .default #endif #if os(iOS) @@ -264,10 +266,6 @@ final class AVVideoIOUnit: NSObject, AVIOUnit { } #endif - @available(macOS, unavailable) - @available(tvOS, unavailable) - var pipCaptureSettings: PiPCaptureSetting = .default - private var lastImageBuffer: CVPixelBuffer? private var pipSampleBuffer: CMSampleBuffer? @@ -334,15 +332,18 @@ final class AVVideoIOUnit: NSObject, AVIOUnit { } @available(iOS 13.0, *) - @available(macOS, unavailable) - func attachPiPCamera(_ camera: AVCaptureDevice?) throws { + func attachMultiCamera(_ camera: AVCaptureDevice?) throws { + #if os(iOS) guard let mixer, AVCaptureMultiCamSession.isMultiCamSupported else { throw Error.multiCamNotSupported } + #else + guard let mixer else { + return + } + #endif guard let camera else { - if mixer.session is AVCaptureMultiCamSession { - mixer.session = AVCaptureSession() - } + mixer.isMultiCamSessionEnabled = false mixer.session.beginConfiguration() defer { mixer.session.commitConfiguration() @@ -350,9 +351,7 @@ final class AVVideoIOUnit: NSObject, AVIOUnit { pipCapture = nil return } - if !(mixer.session is AVCaptureMultiCamSession) { - mixer.session = AVCaptureMultiCamSession() - } + mixer.isMultiCamSessionEnabled = true mixer.session.beginConfiguration() defer { mixer.session.commitConfiguration() @@ -375,7 +374,9 @@ final class AVVideoIOUnit: NSObject, AVIOUnit { if connection.isVideoMirroringSupported { connection.isVideoMirrored = isVideoMirrored } + #if os(iOS) connection.preferredVideoStabilizationMode = preferredVideoStabilizationMode + #endif } pipCapture?.output.setSampleBufferDelegate(self, queue: lockQueue) } @@ -501,7 +502,6 @@ extension AVVideoIOUnit: AVIOUnitDecoding { } } -#if os(iOS) extension AVVideoIOUnit: AVCaptureVideoDataOutputSampleBufferDelegate { // MARK: AVCaptureVideoDataOutputSampleBufferDelegate func captureOutput(_ captureOutput: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { @@ -509,26 +509,17 @@ extension AVVideoIOUnit: AVCaptureVideoDataOutputSampleBufferDelegate { guard mixer?.useSampleBuffer(sampleBuffer: sampleBuffer, mediaType: AVMediaType.video) == true else { return } - appendSampleBuffer(sampleBuffer.over(pipSampleBuffer, regionOfInterest: pipCaptureSettings.regionOfInterest)) + #if os(macOS) + if connection.isVideoMirrored { + sampleBuffer.reflectHorizontal() + } + #endif + appendSampleBuffer(sampleBuffer.over(pipSampleBuffer, regionOfInterest: multiCamCaptureSettings.regionOfInterest)) } else if pipCapture?.output == captureOutput { pipSampleBuffer = sampleBuffer } } } -#elseif os(macOS) -extension AVVideoIOUnit: AVCaptureVideoDataOutputSampleBufferDelegate { - // MARK: AVCaptureVideoDataOutputSampleBufferDelegate - func captureOutput(_ captureOutput: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { - guard mixer?.useSampleBuffer(sampleBuffer: sampleBuffer, mediaType: AVMediaType.video) == true else { - return - } - if connection.isVideoMirrored { - sampleBuffer.reflectHorizontal() - } - appendSampleBuffer(sampleBuffer) - } -} -#endif extension AVVideoIOUnit: VideoCodecDelegate { // MARK: VideoCodecDelegate diff --git a/Sources/Media/PiPCaptureSetting.swift b/Sources/Media/MultiCamCaptureSetting.swift similarity index 63% rename from Sources/Media/PiPCaptureSetting.swift rename to Sources/Media/MultiCamCaptureSetting.swift index 3c99298c..e8bccbb2 100644 --- a/Sources/Media/PiPCaptureSetting.swift +++ b/Sources/Media/MultiCamCaptureSetting.swift @@ -1,11 +1,10 @@ +#if os(iOS) || os(macOS) import CoreMedia import Foundation -/// The PiPCaptureSetting represents the pip capture settings for the video capture. -@available(macOS, unavailable) -@available(tvOS, unavailable) -public struct PiPCaptureSetting { - public static let `default` = PiPCaptureSetting( +/// The MultiCamCaptureSetting represents the pip capture settings for the video capture. +public struct MultiCamCaptureSetting { + public static let `default` = MultiCamCaptureSetting( regionOfInterest: .init( origin: CGPoint(x: 16, y: 16), size: .init(width: 160, height: 160) @@ -20,3 +19,4 @@ public struct PiPCaptureSetting { self.regionOfInterest = regionOfInterest } } +#endif diff --git a/Sources/Net/NetStream.swift b/Sources/Net/NetStream.swift index 4b48e749..8d2c689b 100644 --- a/Sources/Net/NetStream.swift +++ b/Sources/Net/NetStream.swift @@ -117,15 +117,13 @@ open class NetStream: NSObject { } } - /// Specifies the picture in picuture capture protperties. - @available(macOS, unavailable) - @available(tvOS, unavailable) - public var pipCaptureSettings: PiPCaptureSetting { + /// Specifies the multi camera capture properties. + public var multiCamCaptureSettings: MultiCamCaptureSetting { get { - mixer.videoIO.pipCaptureSettings + mixer.videoIO.multiCamCaptureSettings } set { - mixer.videoIO.pipCaptureSettings = newValue + mixer.videoIO.multiCamCaptureSettings = newValue } } @@ -149,11 +147,10 @@ open class NetStream: NSObject { /// Attaches the camera object for picture in picture. /// - Warning: This method can't use appendSampleBuffer at the same time. @available(iOS 13.0, *) - @available(macOS, unavailable) - open func attachPiPCamera(_ camera: AVCaptureDevice?, onError: ((_ error: Error) -> Void)? = nil) { + open func attachMultiCamera(_ camera: AVCaptureDevice?, onError: ((_ error: Error) -> Void)? = nil) { lockQueue.async { do { - try self.mixer.videoIO.attachPiPCamera(camera) + try self.mixer.videoIO.attachMultiCamera(camera) } catch { onError?(error) }