diff --git a/Examples/SimpleFourSquareVideo/SimpleFourSquareVideo.xcodeproj/project.pbxproj b/Examples/SimpleFourSquareVideo/SimpleFourSquareVideo.xcodeproj/project.pbxproj index 46f0612..7b6af40 100644 --- a/Examples/SimpleFourSquareVideo/SimpleFourSquareVideo.xcodeproj/project.pbxproj +++ b/Examples/SimpleFourSquareVideo/SimpleFourSquareVideo.xcodeproj/project.pbxproj @@ -367,7 +367,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; - DEVELOPMENT_TEAM = RYRPKMVKDL; + DEVELOPMENT_TEAM = 3BJYKWHMM4; INFOPLIST_FILE = SimpleFourSquareVideo/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", @@ -385,7 +385,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; - DEVELOPMENT_TEAM = RYRPKMVKDL; + DEVELOPMENT_TEAM = 3BJYKWHMM4; INFOPLIST_FILE = SimpleFourSquareVideo/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", diff --git a/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback.xcodeproj/project.pbxproj b/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback.xcodeproj/project.pbxproj index dbac80d..a23ce27 100644 --- a/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback.xcodeproj/project.pbxproj +++ b/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback.xcodeproj/project.pbxproj @@ -7,6 +7,7 @@ objects = { /* Begin PBXBuildFile section */ + 36E063AD2523E25200CCC13B /* PlaygroundVC.swift in Sources */ = {isa = PBXBuildFile; fileRef = 36E063AC2523E25200CCC13B /* PlaygroundVC.swift */; }; 6ADB075A24ADDB0C0010A817 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6ADB075924ADDB0C0010A817 /* AppDelegate.swift */; }; 6ADB075C24ADDB0C0010A817 /* SceneDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6ADB075B24ADDB0C0010A817 /* SceneDelegate.swift */; }; 6ADB075E24ADDB0C0010A817 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6ADB075D24ADDB0C0010A817 /* ViewController.swift */; }; @@ -51,6 +52,7 @@ /* End PBXCopyFilesBuildPhase section */ /* Begin PBXFileReference section */ + 36E063AC2523E25200CCC13B /* PlaygroundVC.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PlaygroundVC.swift; sourceTree = ""; }; 6ADB075624ADDB0C0010A817 /* SimpleRealtimeFilterPlayback.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = SimpleRealtimeFilterPlayback.app; sourceTree = BUILT_PRODUCTS_DIR; }; 6ADB075924ADDB0C0010A817 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 6ADB075B24ADDB0C0010A817 /* SceneDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SceneDelegate.swift; sourceTree = ""; }; @@ -100,6 +102,7 @@ 6ADB075924ADDB0C0010A817 /* AppDelegate.swift */, 6ADB075B24ADDB0C0010A817 /* SceneDelegate.swift */, 6ADB075D24ADDB0C0010A817 /* ViewController.swift */, + 36E063AC2523E25200CCC13B /* PlaygroundVC.swift */, 6ADB075F24ADDB0C0010A817 /* Main.storyboard */, 6ADB07DB24AED8910010A817 /* 853.mp4 */, 6ADB07D924AED8220010A817 /* cute.mp4 */, @@ -219,6 +222,7 @@ 6ADB075E24ADDB0C0010A817 /* ViewController.swift in Sources */, 6ADB075A24ADDB0C0010A817 /* AppDelegate.swift in Sources */, 6ADB075C24ADDB0C0010A817 /* SceneDelegate.swift in Sources */, + 36E063AD2523E25200CCC13B /* PlaygroundVC.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -371,7 +375,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; - DEVELOPMENT_TEAM = RYRPKMVKDL; + DEVELOPMENT_TEAM = 3BJYKWHMM4; INFOPLIST_FILE = SimpleRealtimeFilterPlayback/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", @@ -389,7 +393,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; - DEVELOPMENT_TEAM = RYRPKMVKDL; + DEVELOPMENT_TEAM = 3BJYKWHMM4; INFOPLIST_FILE = SimpleRealtimeFilterPlayback/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", diff --git a/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback/Base.lproj/Main.storyboard b/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback/Base.lproj/Main.storyboard index 3945b01..54354c3 100644 --- a/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback/Base.lproj/Main.storyboard +++ b/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback/Base.lproj/Main.storyboard @@ -1,12 +1,30 @@ - + - + + + + + + + + + + + + + + + + + + + @@ -16,94 +34,95 @@ - - + + - + - - + - + - + - + - + - - + + @@ -136,8 +155,8 @@ - + @@ -145,7 +164,111 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback/Info.plist b/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback/Info.plist index 2a3483c..677e378 100644 --- a/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback/Info.plist +++ b/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback/Info.plist @@ -20,6 +20,8 @@ 1 LSRequiresIPhoneOS + NSCameraUsageDescription + Use UIApplicationSceneManifest UIApplicationSupportsMultipleScenes diff --git a/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback/PlaygroundVC.swift b/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback/PlaygroundVC.swift new file mode 100644 index 0000000..9fc10b7 --- /dev/null +++ b/Examples/SimpleRealtimeFilterPlayback/SimpleRealtimeFilterPlayback/PlaygroundVC.swift @@ -0,0 +1,91 @@ +// +// PlaygroundVC.swift +// SimpleRealtimeFilterPlayback +// +// Created by wangrenzhu on 2020/9/30. +// Copyright © 2020 RenZhu Macro. All rights reserved. +// + +import UIKit +import AVFoundation +import MetalVideoProcess + + +class PlaygroundVC: UIViewController { + + @IBOutlet weak var renderView: MetalVideoProcessRenderView! + @IBOutlet weak var progress: UISlider! + + var grayFilter: MetalVideoProcessLuminance? + var player: MetalVideoProcessPlayer? + + override func viewDidLoad() { + super.viewDidLoad() + let asset1 = AVAsset(url: Bundle.main.url(forResource: "853", withExtension: "mp4")!) + let item1 = MetalVideoEditorItem(asset: asset1) + + do { + let editor = try MetalVideoEditor(videoItems: [item1], + customVideoCompositorClass: MetalVideoProcessCompositor.self) + + let playerItem = editor.buildPlayerItem() + self.progress.maximumValue = Float(playerItem.duration.seconds) + let player = try MetalVideoProcessPlayer(playerItem: playerItem) + + let gray = MetalVideoProcessLuminance() + gray.saveUniformSettings(forTimelineRange: CMTimeRange(start: CMTime(seconds: 0.5), duration: CMTime(seconds: 3.0)), trackID: item1.trackID) + gray.isEnable = true + self.grayFilter = gray + + player.addTarget(gray, atTargetIndex: nil, trackID: item1.trackID, targetTrackId: 0) + gray --> renderView + + player.playerDelegate = self + self.player = player + } catch { + + } + + } + + override func viewDidDisappear(_ animated: Bool) { + self.player?.suspend() + self.player?.dispose() + } + + @IBAction func play(_ sender: Any) { + self.player?.play() + } + + @IBAction func pause(_ sender: Any) { + self.player?.pause() + } + + @IBAction func progressChanged(_ sender: UISlider) { + let value = sender.value + self.player?.seekTo(time: Float64(value)) + } + +} + +extension PlaygroundVC: MetalVideoProcessPlayerDelegate { + func playbackFrameTimeChanged(frameTime time: CMTime, player: AVPlayer) { + DispatchQueue.main.async { + self.progress.value = Float(time.seconds) + } + } + + func playEnded(currentPlayer player: AVPlayer) { + + } + + func finishExport(error: NSError?) { + + } + + func exportProgressChanged(_ progress: Float) { + + } + + +} diff --git a/Examples/SimpleVideoEditor/SimpleVideoEditor.xcodeproj/project.pbxproj b/Examples/SimpleVideoEditor/SimpleVideoEditor.xcodeproj/project.pbxproj index 625042e..dc7262d 100644 --- a/Examples/SimpleVideoEditor/SimpleVideoEditor.xcodeproj/project.pbxproj +++ b/Examples/SimpleVideoEditor/SimpleVideoEditor.xcodeproj/project.pbxproj @@ -572,7 +572,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; - DEVELOPMENT_TEAM = RYRPKMVKDL; + DEVELOPMENT_TEAM = 3BJYKWHMM4; INFOPLIST_FILE = SimpleVideoEditor/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", @@ -591,7 +591,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; - DEVELOPMENT_TEAM = RYRPKMVKDL; + DEVELOPMENT_TEAM = 3BJYKWHMM4; INFOPLIST_FILE = SimpleVideoEditor/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", diff --git a/Examples/SimpleVideoEditor/SimpleVideoEditor/ViewController+MetalVideoProcessPlayerDelegate.swift b/Examples/SimpleVideoEditor/SimpleVideoEditor/ViewController+MetalVideoProcessPlayerDelegate.swift index 6d8c65f..992c427 100644 --- a/Examples/SimpleVideoEditor/SimpleVideoEditor/ViewController+MetalVideoProcessPlayerDelegate.swift +++ b/Examples/SimpleVideoEditor/SimpleVideoEditor/ViewController+MetalVideoProcessPlayerDelegate.swift @@ -136,8 +136,9 @@ extension ViewController: MetalVideoProcessPlayerDelegate { source --> self.movieWriter! self.view.isUserInteractionEnabled = false - try? self.player?.startExport() + self.movieWriter?.activateAudioTrack() self.movieWriter?.startRecording() + try? self.player?.startExport() } } diff --git a/Examples/SimpleVideoExport/SimpleVideoExport.xcodeproj/project.pbxproj b/Examples/SimpleVideoExport/SimpleVideoExport.xcodeproj/project.pbxproj index 4eaaeb3..6a26d4d 100644 --- a/Examples/SimpleVideoExport/SimpleVideoExport.xcodeproj/project.pbxproj +++ b/Examples/SimpleVideoExport/SimpleVideoExport.xcodeproj/project.pbxproj @@ -385,7 +385,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; - DEVELOPMENT_TEAM = RYRPKMVKDL; + DEVELOPMENT_TEAM = 3BJYKWHMM4; INFOPLIST_FILE = SimpleVideoExport/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = ( @@ -405,7 +405,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; - DEVELOPMENT_TEAM = RYRPKMVKDL; + DEVELOPMENT_TEAM = 3BJYKWHMM4; INFOPLIST_FILE = SimpleVideoExport/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = ( diff --git a/Examples/SimpleVideoExport/SimpleVideoExport/ViewController.swift b/Examples/SimpleVideoExport/SimpleVideoExport/ViewController.swift index 7b64000..2356a7d 100644 --- a/Examples/SimpleVideoExport/SimpleVideoExport/ViewController.swift +++ b/Examples/SimpleVideoExport/SimpleVideoExport/ViewController.swift @@ -118,9 +118,10 @@ class ViewController: UIViewController { source --> self.movieWriter! self.view.isUserInteractionEnabled = false - try? self.player?.startExport() + self.movieWriter?.activateAudioTrack() self.movieWriter?.startRecording() + try? self.player?.startExport() } diff --git a/Examples/SimpleVideoTransition/SimpleVideoTransition.xcodeproj/project.pbxproj b/Examples/SimpleVideoTransition/SimpleVideoTransition.xcodeproj/project.pbxproj index 7a9d806..eafc132 100644 --- a/Examples/SimpleVideoTransition/SimpleVideoTransition.xcodeproj/project.pbxproj +++ b/Examples/SimpleVideoTransition/SimpleVideoTransition.xcodeproj/project.pbxproj @@ -361,7 +361,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; - DEVELOPMENT_TEAM = RYRPKMVKDL; + DEVELOPMENT_TEAM = 3BJYKWHMM4; INFOPLIST_FILE = SimpleVideoTransition/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = ( @@ -380,7 +380,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; - DEVELOPMENT_TEAM = RYRPKMVKDL; + DEVELOPMENT_TEAM = 3BJYKWHMM4; INFOPLIST_FILE = SimpleVideoTransition/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = ( diff --git a/MetalVideoProcess.xcodeproj/project.pbxproj b/MetalVideoProcess.xcodeproj/project.pbxproj index 6d7606a..00ef1dc 100644 --- a/MetalVideoProcess.xcodeproj/project.pbxproj +++ b/MetalVideoProcess.xcodeproj/project.pbxproj @@ -990,7 +990,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; DEFINES_MODULE = YES; - DEVELOPMENT_TEAM = RYRPKMVKDL; + DEVELOPMENT_TEAM = ""; DYLIB_COMPATIBILITY_VERSION = 1; DYLIB_CURRENT_VERSION = 1; DYLIB_INSTALL_NAME_BASE = "@rpath"; @@ -1017,7 +1017,7 @@ CLANG_ENABLE_MODULES = YES; CODE_SIGN_STYLE = Automatic; DEFINES_MODULE = YES; - DEVELOPMENT_TEAM = RYRPKMVKDL; + DEVELOPMENT_TEAM = ""; DYLIB_COMPATIBILITY_VERSION = 1; DYLIB_CURRENT_VERSION = 1; DYLIB_INSTALL_NAME_BASE = "@rpath"; diff --git a/MetalVideoProcess/Vender/Render/Inputs/Camera.swift b/MetalVideoProcess/Vender/Render/Inputs/Camera.swift index 8891cdc..352c2d2 100644 --- a/MetalVideoProcess/Vender/Render/Inputs/Camera.swift +++ b/MetalVideoProcess/Vender/Render/Inputs/Camera.swift @@ -1,6 +1,7 @@ import Foundation import AVFoundation import Metal +import CoreImage public protocol CameraDelegate { func didCaptureBuffer(_ sampleBuffer: CMSampleBuffer) @@ -72,7 +73,10 @@ public class Camera: NSObject, ImageSource, AVCaptureVideoDataOutputSampleBuffer var supportsFullYUVRange: Bool = false let captureAsYUV: Bool - let yuvConversionRenderPipelineState: MTLRenderPipelineState? +// let yuvConversionRenderPipelineState: MTLRenderPipelineState? + + /// 通过Compute方式转化 + var yuvConversionComputePipelineState: MTLComputePipelineState? var yuvLookupTable: [String: (Int, MTLDataType)] = [: ] let frameRenderingSemaphore = DispatchSemaphore(value: 1) @@ -110,7 +114,7 @@ public class Camera: NSObject, ImageSource, AVCaptureVideoDataOutputSampleBuffer self.videoInput = nil self.videoOutput = nil self.inputCamera = nil - self.yuvConversionRenderPipelineState = nil + self.yuvConversionComputePipelineState = nil super.init() throw CameraError() } @@ -121,7 +125,7 @@ public class Camera: NSObject, ImageSource, AVCaptureVideoDataOutputSampleBuffer } catch { self.videoInput = nil self.videoOutput = nil - self.yuvConversionRenderPipelineState = nil + self.yuvConversionComputePipelineState = nil super.init() throw error } @@ -133,7 +137,12 @@ public class Camera: NSObject, ImageSource, AVCaptureVideoDataOutputSampleBuffer // Add the video frame output videoOutput = AVCaptureVideoDataOutput() videoOutput.alwaysDiscardsLateVideoFrames = false - + #if targetEnvironment(simulator) + self.yuvConversionRenderPipelineState = nil + super.init() + #else + + if captureAsYUV { supportsFullYUVRange = false let supportedPixelFormats = videoOutput.availableVideoPixelFormatTypes @@ -143,20 +152,20 @@ public class Camera: NSObject, ImageSource, AVCaptureVideoDataOutputSampleBuffer } } if (supportsFullYUVRange) { - let (pipelineState, lookupTable) = generateRenderPipelineState(device: sharedMetalRenderingDevice, vertexFunctionName: "twoInputVertex", fragmentFunctionName: "yuvConversionFullRangeFragment", operationName: "YUVToRGB") - self.yuvConversionRenderPipelineState = pipelineState + let (pipelineState, lookupTable) = generateComputePipelineState(device: sharedMetalRenderingDevice, kernelFunctionName: "yuv2rgb", operationName: "YUVToRGB") self.yuvLookupTable = lookupTable + self.yuvConversionComputePipelineState = pipelineState videoOutput.videoSettings = [kCVPixelBufferMetalCompatibilityKey as String: true, kCVPixelBufferPixelFormatTypeKey as String: NSNumber(value: Int32(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange))] } else { - let (pipelineState, lookupTable) = generateRenderPipelineState(device: sharedMetalRenderingDevice, vertexFunctionName: "twoInputVertex", fragmentFunctionName: "yuvConversionVideoRangeFragment", operationName: "YUVToRGB") - self.yuvConversionRenderPipelineState = pipelineState - self.yuvLookupTable = lookupTable - videoOutput.videoSettings = [kCVPixelBufferMetalCompatibilityKey as String: true, - kCVPixelBufferPixelFormatTypeKey as String: NSNumber(value: Int32(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange))] +// let (pipelineState, lookupTable) = generateRenderPipelineState(device: sharedMetalRenderingDevice, vertexFunctionName: "twoInputVertex", fragmentFunctionName: "yuvConversionVideoRangeFragment", operationName: "YUVToRGB") +// self.yuvConversionRenderPipelineState = pipelineState +// self.yuvLookupTable = lookupTable +// videoOutput.videoSettings = [kCVPixelBufferMetalCompatibilityKey as String: true, +// kCVPixelBufferPixelFormatTypeKey as String: NSNumber(value: Int32(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange))] } } else { - self.yuvConversionRenderPipelineState = nil + self.yuvConversionComputePipelineState = nil videoOutput.videoSettings = [kCVPixelBufferMetalCompatibilityKey as String: true, kCVPixelBufferPixelFormatTypeKey as String: NSNumber(value: Int32(kCVPixelFormatType_32BGRA))] } @@ -173,6 +182,7 @@ public class Camera: NSObject, ImageSource, AVCaptureVideoDataOutputSampleBuffer let _ = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, sharedMetalRenderingDevice.device, nil, &videoTextureCache) videoOutput.setSampleBufferDelegate(self, queue: cameraProcessingQueue) + #endif } deinit { @@ -228,11 +238,16 @@ public class Camera: NSObject, ImageSource, AVCaptureVideoDataOutputSampleBuffer outputHeight = bufferHeight } let outputTexture = Texture(device: sharedMetalRenderingDevice.device, orientation: .portrait, width: outputWidth, height: outputHeight, timingStyle: .videoFrame(timestamp: Timestamp(currentTime))) - - convertYUVToRGB(pipelineState: self.yuvConversionRenderPipelineState!, lookupTable: self.yuvLookupTable, - luminanceTexture: Texture(orientation: self.orientation ?? self.location.imageOrientation(), texture: luminanceTexture), - chrominanceTexture: Texture(orientation: self.orientation ?? self.location.imageOrientation(), texture: chrominanceTexture), - resultTexture: outputTexture, colorConversionMatrix: conversionMatrix) + convertYUVtoRGBCompute(pipelineState: self.yuvConversionComputePipelineState!, + lookupTable: self.yuvLookupTable, + luminanceTexture: Texture(orientation: self.orientation ?? self.location.imageOrientation(), texture: luminanceTexture), + chrominanceTexture: Texture(orientation: self.orientation ?? self.location.imageOrientation(), texture: chrominanceTexture), + resultTexture: outputTexture, + colorConversionMatrix: conversionMatrix) +// convertYUVToRGB(pipelineState: self.yuvConversionRenderPipelineState!, lookupTable: self.yuvLookupTable, +// luminanceTexture: Texture(orientation: self.orientation ?? self.location.imageOrientation(), texture: luminanceTexture), +// chrominanceTexture: Texture(orientation: self.orientation ?? self.location.imageOrientation(), texture: chrominanceTexture), +// resultTexture: outputTexture, colorConversionMatrix: conversionMatrix) texture = outputTexture } else { texture = nil diff --git a/MetalVideoProcess/Vender/Render/Inputs/YUVToRGBConversion.metal b/MetalVideoProcess/Vender/Render/Inputs/YUVToRGBConversion.metal index 2f7166d..68b56d6 100644 --- a/MetalVideoProcess/Vender/Render/Inputs/YUVToRGBConversion.metal +++ b/MetalVideoProcess/Vender/Render/Inputs/YUVToRGBConversion.metal @@ -36,3 +36,22 @@ fragment half4 yuvConversionVideoRangeFragment(TwoInputVertexIO fragmentInput [[ return half4(rgb, 1.0); } + +constant float3 ColorOffsetFullRange = float3(0.0, -0.5, -0.5); + +kernel void yuv2rgb(texture2d y_tex [[ texture(0) ]], + texture2d uv_tex [[ texture(1) ]], + texture2d bgr_tex [[ texture(2) ]], + uint2 gid [[thread_position_in_grid]]) +{ + + float3 yuv = float3(y_tex.read(gid).r, uv_tex.read(gid/2).rg) + ColorOffsetFullRange; + + const float3x3 kColorConversion601Default = { + {1.164, 1.164, 1.164}, + {0.0, -0.392, 2.017}, + {1.596, -0.813, 0.0}, + }; + + bgr_tex.write(float4(float3(kColorConversion601Default * yuv), 1.0), gid); +} diff --git a/MetalVideoProcess/Vender/Render/Inputs/YUVToRGBConversion.swift b/MetalVideoProcess/Vender/Render/Inputs/YUVToRGBConversion.swift index fda25e0..d3c5e68 100644 --- a/MetalVideoProcess/Vender/Render/Inputs/YUVToRGBConversion.swift +++ b/MetalVideoProcess/Vender/Render/Inputs/YUVToRGBConversion.swift @@ -21,6 +21,30 @@ public let colorConversionMatrix709Default = Matrix3x3(rowMajorValues: [ 1.793, -0.533, 0.0, ]) +public func convertYUVtoRGBCompute(pipelineState: MTLComputePipelineState, + lookupTable: [String:(Int, MTLDataType)], + luminanceTexture: Texture, + chrominanceTexture: Texture, + secondChrominanceTexture: Texture? = nil, + resultTexture: Texture, + colorConversionMatrix: Matrix3x3, + device: MetalRenderingDevice = sharedMetalRenderingDevice) { + + guard let commandBuffer = device.commandQueue.makeCommandBuffer() else {return} + + let inputTextures: [UInt: Texture] + if let secondChrominanceTexture = secondChrominanceTexture { + inputTextures = [0: luminanceTexture, 1: chrominanceTexture, 2: secondChrominanceTexture] + } else { + inputTextures = [0: luminanceTexture, 1: chrominanceTexture] + } + + let inputWidth = resultTexture.texture.width + let inputHeight = resultTexture.texture.height + commandBuffer.computeQuad(pipelineState: pipelineState, uniformSettings: nil, inputTextures: inputTextures, outputTexture: resultTexture, threadGroups: MTLSizeMake(inputWidth / 16, inputHeight / 16, 1)) + commandBuffer.commit() +} + public func convertYUVToRGB(pipelineState: MTLRenderPipelineState, lookupTable: [String: (Int, MTLDataType)], luminanceTexture: Texture, chrominanceTexture: Texture, secondChrominanceTexture: Texture? = nil, resultTexture: Texture, colorConversionMatrix: Matrix3x3) { let uniformSettings = ShaderUniformSettings(uniformLookupTable: lookupTable) uniformSettings["colorConversionMatrix"] = colorConversionMatrix