GitHub/HaishinKit.swift
1
0
Fork 0
mirror of https://github.com/HaishinKit/HaishinKit.swift.git synced 2026-06-06 20:17:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support multi camera capture on macOS.

Browse Source
This commit is contained in:
shogo4405
2022-12-03 18:54:32 +09:00
parent df6a1bd8ca
commit 26e1606f10
7 changed files with 64 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Examples/iOS/LiveViewController.swift
+3 -3
View File
@@ -32,7 +32,7 @@ final class LiveViewController: UIViewController {
pipIntentView.layer.borderWidth = 1.0
pipIntentView.layer.borderColor = UIColor.white.cgColor
pipIntentView.bounds = PiPCaptureSetting.default.regionOfInterest
pipIntentView.bounds = MultiCamCaptureSetting.default.regionOfInterest
pipIntentView.isUserInteractionEnabled = true
view.addSubview(pipIntentView)
@@ -72,7 +72,7 @@ final class LiveViewController: UIViewController {
}
if #available(iOS 13.0, *) {
let front = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .front)
rtmpStream.attachPiPCamera(front)
rtmpStream.attachMultiCamera(front)
}
rtmpStream.addObserver(self, forKeyPath: "currentFPS", options: .new, context: nil)
lfView?.attachStream(rtmpStream)
@@ -109,7 +109,7 @@ final class LiveViewController: UIViewController {
currentFrame.origin.x += deltaX
currentFrame.origin.y += deltaY
pipIntentView.frame = currentFrame
rtmpStream.pipCaptureSettings = PiPCaptureSetting(regionOfInterest: currentFrame)
rtmpStream.multiCamCaptureSettings = MultiCamCaptureSetting(regionOfInterest: currentFrame)
}
}
HaishinKit.xcodeproj/project.pbxproj
+8 -8
View File
@@ -330,9 +330,9 @@
BC110253292DD6E900D48035 /* vImage_Buffer+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110252292DD6E900D48035 /* vImage_Buffer+Extension.swift */; };
BC110254292DD6E900D48035 /* vImage_Buffer+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110252292DD6E900D48035 /* vImage_Buffer+Extension.swift */; };
BC110255292DD6E900D48035 /* vImage_Buffer+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110252292DD6E900D48035 /* vImage_Buffer+Extension.swift */; };
BC110257292E661E00D48035 /* PiPCaptureSetting.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110256292E661E00D48035 /* PiPCaptureSetting.swift */; };
BC110258292E661E00D48035 /* PiPCaptureSetting.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110256292E661E00D48035 /* PiPCaptureSetting.swift */; };
BC110259292E661E00D48035 /* PiPCaptureSetting.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110256292E661E00D48035 /* PiPCaptureSetting.swift */; };
BC110257292E661E00D48035 /* MultiCamCaptureSetting.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110256292E661E00D48035 /* MultiCamCaptureSetting.swift */; };
BC110258292E661E00D48035 /* MultiCamCaptureSetting.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110256292E661E00D48035 /* MultiCamCaptureSetting.swift */; };
BC110259292E661E00D48035 /* MultiCamCaptureSetting.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC110256292E661E00D48035 /* MultiCamCaptureSetting.swift */; };
BC11D94625A1B01000D710BA /* Screencast.appex in Embed Foundation Extensions */ = {isa = PBXBuildFile; fileRef = 2915EC521D85BDF100621092 /* Screencast.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; };
BC20DF38250377A3007BC608 /* ScreenCaptureSession.swift in Sources */ = {isa = PBXBuildFile; fileRef = 299B131C1D35272D00A1E8F5 /* ScreenCaptureSession.swift */; };
BC34DFF025EBB53F005F975A /* Logboard.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = BC34DFD125EBB12C005F975A /* Logboard.xcframework */; };
@@ -849,7 +849,7 @@
BC11023D2917C35B00D48035 /* CVPixelBufferPool+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CVPixelBufferPool+Extension.swift"; sourceTree = "<group>"; };
BC1102492925147300D48035 /* AVCaptureIOUnit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AVCaptureIOUnit.swift; sourceTree = "<group>"; };
BC110252292DD6E900D48035 /* vImage_Buffer+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "vImage_Buffer+Extension.swift"; sourceTree = "<group>"; };
BC110256292E661E00D48035 /* PiPCaptureSetting.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PiPCaptureSetting.swift; sourceTree = "<group>"; };
BC110256292E661E00D48035 /* MultiCamCaptureSetting.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MultiCamCaptureSetting.swift; sourceTree = "<group>"; };
BC34DFD125EBB12C005F975A /* Logboard.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = Logboard.xcframework; path = Carthage/Build/Logboard.xcframework; sourceTree = "<group>"; };
BC34FA0A286CB90A00EFAF27 /* PiPHKView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PiPHKView.swift; sourceTree = "<group>"; };
BC44A1A823D31E92002D4297 /* AudioCodecBuffer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioCodecBuffer.swift; sourceTree = "<group>"; wrapsLines = 1; };
@@ -1378,7 +1378,7 @@
BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */,
2901A4ED1D437170002BBD23 /* MediaLink.swift */,
2999C3742071138F00892E55 /* MTHKView.swift */,
BC110256292E661E00D48035 /* PiPCaptureSetting.swift */,
BC110256292E661E00D48035 /* MultiCamCaptureSetting.swift */,
BC34FA0A286CB90A00EFAF27 /* PiPHKView.swift */,
295891001EEB7A8B00CE51E1 /* ScalingMode.swift */,
29B8768D1CD70AFE00FC07DA /* SoundTransform.swift */,
@@ -2118,7 +2118,7 @@
2976A47E1D48C5C700B53EF2 /* AVRecorder.swift in Sources */,
BCC1A6C52643F41600661156 /* MP4MovieFragmentBox.Builder.swift in Sources */,
BC94E4FE263FE6B80094C169 /* MP4MovieFragmentHeaderBox.swift in Sources */,
BC110257292E661E00D48035 /* PiPCaptureSetting.swift in Sources */,
BC110257292E661E00D48035 /* MultiCamCaptureSetting.swift in Sources */,
29B876B21CD70B2800FC07DA /* RTMPMuxer.swift in Sources */,
2958912E1EEB8F4100CE51E1 /* FLVSoundType.swift in Sources */,
BC0D236D26331BAB001DDA0C /* NetSocket.CircularBuffer.swift in Sources */,
@@ -2299,7 +2299,7 @@
BC83A4742403D83B006BDE06 /* VTCompressionSession+Extension.swift in Sources */,
BCC1A72C264FAC1800661156 /* ElementaryStreamSpecificData.swift in Sources */,
29B876FF1CD70D5A00FC07DA /* NALUnit.swift in Sources */,
BC110258292E661E00D48035 /* PiPCaptureSetting.swift in Sources */,
BC110258292E661E00D48035 /* MultiCamCaptureSetting.swift in Sources */,
29B877001CD70D5A00FC07DA /* PacketizedElementaryStream.swift in Sources */,
BCC1A6EC2644850F00661156 /* MP4TrackHeaderBox.swift in Sources */,
BC570B4928E9ACC10098A12C /* AVIOUnit.swift in Sources */,
@@ -2505,7 +2505,7 @@
BCC1A7112647F25A00661156 /* DecoderConfigDescriptor.swift in Sources */,
BCC1A6D126445AF500661156 /* MP4TrackExtendsBox.swift in Sources */,
BC4914A828DDD367009E2DF6 /* VTSessionOption.swift in Sources */,
BC110259292E661E00D48035 /* PiPCaptureSetting.swift in Sources */,
BC110259292E661E00D48035 /* MultiCamCaptureSetting.swift in Sources */,
BC4914A428DDD33D009E2DF6 /* VTSessionConvertible.swift in Sources */,
BCAD0C02263E968400ADFB80 /* MP4FileReader.swift in Sources */,
BCA97C00263C599C0027213C /* MP4SyncSampleBox.swift in Sources */,
Sources/Media/AVAudioIOUnit.swift
+2 -2
View File
@@ -40,7 +40,7 @@ final class AVAudioIOUnit: NSObject, AVIOUnit {
#if os(iOS) || os(macOS)
func attachAudio(_ audio: AVCaptureDevice?, automaticallyConfiguresApplicationAudioSession: Bool) throws {
guard let mixer = mixer else {
guard let mixer else {
return
}
mixer.session.beginConfiguration()
@@ -48,7 +48,7 @@ final class AVAudioIOUnit: NSObject, AVIOUnit {
mixer.session.commitConfiguration()
}
codec.invalidate()
guard let audio: AVCaptureDevice = audio else {
guard let audio else {
capture = nil
return
}
Sources/Media/AVMixer.swift
+21
View File
@@ -74,6 +74,27 @@ public class AVMixer {
case passthrough
}
var isMultiCamSessionEnabled = false {
didSet {
guard oldValue != isMultiCamSessionEnabled else {
return
}
#if os(iOS)
if #available(iOS 13.0, *) {
if isMultiCamSessionEnabled {
if !(session is AVCaptureMultiCamSession) {
session = AVCaptureMultiCamSession()
}
} else {
if session is AVCaptureMultiCamSession {
session = AVCaptureSession()
}
}
}
#endif
}
}
#if os(iOS)
var preferredVideoStabilizationMode: AVCaptureVideoStabilizationMode {
get {
Sources/Media/AVVideoIOUnit.swift
+19 -28
View File
@@ -249,6 +249,8 @@ final class AVVideoIOUnit: NSObject, AVIOUnit {
oldValue?.detach(mixer?.session)
}
}
var multiCamCaptureSettings: MultiCamCaptureSetting = .default
#endif
#if os(iOS)
@@ -264,10 +266,6 @@ final class AVVideoIOUnit: NSObject, AVIOUnit {
}
#endif
@available(macOS, unavailable)
@available(tvOS, unavailable)
var pipCaptureSettings: PiPCaptureSetting = .default
private var lastImageBuffer: CVPixelBuffer?
private var pipSampleBuffer: CMSampleBuffer?
@@ -334,15 +332,18 @@ final class AVVideoIOUnit: NSObject, AVIOUnit {
}
@available(iOS 13.0, *)
@available(macOS, unavailable)
func attachPiPCamera(_ camera: AVCaptureDevice?) throws {
func attachMultiCamera(_ camera: AVCaptureDevice?) throws {
#if os(iOS)
guard let mixer, AVCaptureMultiCamSession.isMultiCamSupported else {
throw Error.multiCamNotSupported
}
#else
guard let mixer else {
return
}
#endif
guard let camera else {
if mixer.session is AVCaptureMultiCamSession {
mixer.session = AVCaptureSession()
}
mixer.isMultiCamSessionEnabled = false
mixer.session.beginConfiguration()
defer {
mixer.session.commitConfiguration()
@@ -350,9 +351,7 @@ final class AVVideoIOUnit: NSObject, AVIOUnit {
pipCapture = nil
return
}
if !(mixer.session is AVCaptureMultiCamSession) {
mixer.session = AVCaptureMultiCamSession()
}
mixer.isMultiCamSessionEnabled = true
mixer.session.beginConfiguration()
defer {
mixer.session.commitConfiguration()
@@ -375,7 +374,9 @@ final class AVVideoIOUnit: NSObject, AVIOUnit {
if connection.isVideoMirroringSupported {
connection.isVideoMirrored = isVideoMirrored
}
#if os(iOS)
connection.preferredVideoStabilizationMode = preferredVideoStabilizationMode
#endif
}
pipCapture?.output.setSampleBufferDelegate(self, queue: lockQueue)
}
@@ -501,7 +502,6 @@ extension AVVideoIOUnit: AVIOUnitDecoding {
}
}
#if os(iOS)
extension AVVideoIOUnit: AVCaptureVideoDataOutputSampleBufferDelegate {
// MARK: AVCaptureVideoDataOutputSampleBufferDelegate
func captureOutput(_ captureOutput: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
@@ -509,26 +509,17 @@ extension AVVideoIOUnit: AVCaptureVideoDataOutputSampleBufferDelegate {
guard mixer?.useSampleBuffer(sampleBuffer: sampleBuffer, mediaType: AVMediaType.video) == true else {
return
}
appendSampleBuffer(sampleBuffer.over(pipSampleBuffer, regionOfInterest: pipCaptureSettings.regionOfInterest))
#if os(macOS)
if connection.isVideoMirrored {
sampleBuffer.reflectHorizontal()
}
#endif
appendSampleBuffer(sampleBuffer.over(pipSampleBuffer, regionOfInterest: multiCamCaptureSettings.regionOfInterest))
} else if pipCapture?.output == captureOutput {
pipSampleBuffer = sampleBuffer
}
}
}
#elseif os(macOS)
extension AVVideoIOUnit: AVCaptureVideoDataOutputSampleBufferDelegate {
// MARK: AVCaptureVideoDataOutputSampleBufferDelegate
func captureOutput(_ captureOutput: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard mixer?.useSampleBuffer(sampleBuffer: sampleBuffer, mediaType: AVMediaType.video) == true else {
return
}
if connection.isVideoMirrored {
sampleBuffer.reflectHorizontal()
}
appendSampleBuffer(sampleBuffer)
}
}
#endif
extension AVVideoIOUnit: VideoCodecDelegate {
// MARK: VideoCodecDelegate
Sources/Media/PiPCaptureSetting.swift → Sources/Media/MultiCamCaptureSetting.swift
+5 -5
View File
@@ -1,11 +1,10 @@
#if os(iOS) || os(macOS)
import CoreMedia
import Foundation
/// The PiPCaptureSetting represents the pip capture settings for the video capture.
@available(macOS, unavailable)
@available(tvOS, unavailable)
public struct PiPCaptureSetting {
public static let `default` = PiPCaptureSetting(
/// The MultiCamCaptureSetting represents the pip capture settings for the video capture.
public struct MultiCamCaptureSetting {
public static let `default` = MultiCamCaptureSetting(
regionOfInterest: .init(
origin: CGPoint(x: 16, y: 16),
size: .init(width: 160, height: 160)
@@ -20,3 +19,4 @@ public struct PiPCaptureSetting {
self.regionOfInterest = regionOfInterest
}
}
#endif
Sources/Net/NetStream.swift
+6 -9
View File
@@ -117,15 +117,13 @@ open class NetStream: NSObject {
}
}
/// Specifies the picture in picuture capture protperties.
@available(macOS, unavailable)
@available(tvOS, unavailable)
public var pipCaptureSettings: PiPCaptureSetting {
/// Specifies the multi camera capture properties.
public var multiCamCaptureSettings: MultiCamCaptureSetting {
get {
mixer.videoIO.pipCaptureSettings
mixer.videoIO.multiCamCaptureSettings
}
set {
mixer.videoIO.pipCaptureSettings = newValue
mixer.videoIO.multiCamCaptureSettings = newValue
}
}
@@ -149,11 +147,10 @@ open class NetStream: NSObject {
/// Attaches the camera object for picture in picture.
/// - Warning: This method can't use appendSampleBuffer at the same time.
@available(iOS 13.0, *)
@available(macOS, unavailable)
open func attachPiPCamera(_ camera: AVCaptureDevice?, onError: ((_ error: Error) -> Void)? = nil) {
open func attachMultiCamera(_ camera: AVCaptureDevice?, onError: ((_ error: Error) -> Void)? = nil) {
lockQueue.async {
do {
try self.mixer.videoIO.attachPiPCamera(camera)
try self.mixer.videoIO.attachMultiCamera(camera)
} catch {
onError?(error)
}