diff --git a/.github/workflows/asr-benchmark.yml b/.github/workflows/asr-benchmark.yml index 73b75a6e..9fe61e72 100644 --- a/.github/workflows/asr-benchmark.yml +++ b/.github/workflows/asr-benchmark.yml @@ -20,14 +20,19 @@ jobs: with: swift-version: "6.1" + - name: Install ffmpeg + run: | + brew install ffmpeg || echo "ffmpeg may already be installed" + ffmpeg -version || echo "ffmpeg not available" + - name: Cache Dependencies uses: actions/cache@v4 with: path: | .build ~/Library/Application Support/FluidAudio/Models/Parakeet - ~/Documents/Datasets/librispeech - key: ${{ runner.os }}-asr-${{ hashFiles('Package.resolved') }}-v4 + ~/Library/Application Support/FluidAudio/Datasets/LibriSpeech + key: ${{ runner.os }}-asr-${{ hashFiles('Package.resolved') }}-v5 - name: Build run: swift build -c release @@ -69,25 +74,44 @@ jobs: MAX_FILES="25" BENCHMARK_START=$(date +%s) - # Run standard benchmarks in parallel - swift run -c release fluidaudio asr-benchmark \ - --subset test-clean --max-files "$MAX_FILES" \ - --auto-download --output asr_results_clean.json & - CLEAN_PID=$! + # Set error handling + set -o pipefail - swift run -c release fluidaudio asr-benchmark \ - --subset test-other --max-files "$MAX_FILES" \ - --auto-download --output asr_results_other.json & - OTHER_PID=$! + # Function to run benchmark with error capture + run_benchmark() { + local SUBSET=$1 + local MAX=$2 + local OUTPUT=$3 + local EXTRA_ARGS="${4:-}" + + echo "=========================================" + echo "Running ASR benchmark: $SUBSET (max $MAX files)" + echo "Output: $OUTPUT" + echo "Extra args: $EXTRA_ARGS" + echo "=========================================" + + if swift run -c release fluidaudio asr-benchmark \ + --subset "$SUBSET" --max-files "$MAX" \ + --auto-download --output "$OUTPUT" $EXTRA_ARGS 2>&1 | tee benchmark_log.txt; then + echo "✅ Benchmark $SUBSET completed successfully" + return 0 + else + echo "❌ Benchmark $SUBSET FAILED with exit code $?" + echo "Last 50 lines of output:" + tail -50 benchmark_log.txt + # Continue with other benchmarks even if one fails + return 1 + fi + } + + # Run benchmarks with error capture + run_benchmark "test-clean" "$MAX_FILES" "asr_results_clean.json" || CLEAN_FAILED=1 + + run_benchmark "test-other" "$MAX_FILES" "asr_results_other.json" || OTHER_FAILED=1 # Run streaming benchmark (smaller file count for faster CI) - swift run -c release fluidaudio asr-benchmark \ - --subset test-clean --max-files "5" \ - --test-streaming --chunk-duration 0.5 \ - --auto-download --output asr_results_streaming.json & - STREAMING_PID=$! + run_benchmark "test-clean" "5" "asr_results_streaming.json" "--test-streaming --chunk-duration 0.5" || STREAMING_FAILED=1 - wait $CLEAN_PID && wait $OTHER_PID && wait $STREAMING_PID # Extract metrics with error handling if [ -f asr_results_clean.json ]; then @@ -152,18 +176,37 @@ jobs: echo "EXECUTION_TIME=$EXECUTION_TIME" >> $GITHUB_OUTPUT echo "FILES_COUNT=$MAX_FILES" >> $GITHUB_OUTPUT + # Report failures summary + if [ ! -z "$CLEAN_FAILED" ] || [ ! -z "$OTHER_FAILED" ] || [ ! -z "$STREAMING_FAILED" ]; then + echo "BENCHMARK_STATUS=PARTIAL_FAILURE" >> $GITHUB_OUTPUT + echo "⚠️ Some benchmarks failed:" + [ ! -z "$CLEAN_FAILED" ] && echo " - test-clean benchmark failed" + [ ! -z "$OTHER_FAILED" ] && echo " - test-other benchmark failed" + [ ! -z "$STREAMING_FAILED" ] && echo " - streaming benchmark failed" + # Don't exit with error to allow PR comment to be posted + else + echo "BENCHMARK_STATUS=SUCCESS" >> $GITHUB_OUTPUT + echo "✅ All benchmarks completed successfully" + fi + - name: Comment PR if: github.event_name == 'pull_request' continue-on-error: true uses: actions/github-script@v7 with: script: | - const body = `## ASR Benchmark Results + const benchmarkStatus = '${{ steps.benchmark.outputs.BENCHMARK_STATUS }}'; + const statusEmoji = benchmarkStatus === 'SUCCESS' ? '✅' : '⚠️'; + const statusText = benchmarkStatus === 'SUCCESS' ? 'All benchmarks passed' : 'Some benchmarks failed (see logs)'; + + const body = `## ASR Benchmark Results ${statusEmoji} + + **Status:** ${statusText} | Dataset | WER Avg | WER Med | RTFx | Status | |---------|---------|---------|------|--------| - | test-clean | ${{ steps.benchmark.outputs.CLEAN_WER_AVG }}% | ${{ steps.benchmark.outputs.CLEAN_WER_MED }}% | ${{ steps.benchmark.outputs.CLEAN_RTFx }}x | ${parseFloat('${{ steps.benchmark.outputs.CLEAN_WER_AVG }}') < 10 ? '✅' : '⚠️'} | - | test-other | ${{ steps.benchmark.outputs.OTHER_WER_AVG }}% | ${{ steps.benchmark.outputs.OTHER_WER_MED }}% | ${{ steps.benchmark.outputs.OTHER_RTFx }}x | ${parseFloat('${{ steps.benchmark.outputs.OTHER_WER_AVG }}') < 20 ? '✅' : '⚠️'} | + | test-clean | ${{ steps.benchmark.outputs.CLEAN_WER_AVG }}% | ${{ steps.benchmark.outputs.CLEAN_WER_MED }}% | ${{ steps.benchmark.outputs.CLEAN_RTFx }}x | ${parseFloat('${{ steps.benchmark.outputs.CLEAN_WER_AVG }}') < 10 ? '✅' : '${{ steps.benchmark.outputs.CLEAN_WER_AVG }}' === 'N/A' ? '❌' : '⚠️'} | + | test-other | ${{ steps.benchmark.outputs.OTHER_WER_AVG }}% | ${{ steps.benchmark.outputs.OTHER_WER_MED }}% | ${{ steps.benchmark.outputs.OTHER_RTFx }}x | ${parseFloat('${{ steps.benchmark.outputs.OTHER_WER_AVG }}') < 20 ? '✅' : '${{ steps.benchmark.outputs.OTHER_WER_AVG }}' === 'N/A' ? '❌' : '⚠️'} | ### Streaming Infrastructure Test | Metric | Value | Description | diff --git a/Sources/FluidAudio/ASR/AsrManager.swift b/Sources/FluidAudio/ASR/AsrManager.swift index 3ddb2a06..ae60c291 100644 --- a/Sources/FluidAudio/ASR/AsrManager.swift +++ b/Sources/FluidAudio/ASR/AsrManager.swift @@ -145,39 +145,6 @@ public final class AsrManager { ]) } - func prepareMelSpectrogramInputFP16( - _ audioSamples: [Float], actualLength: Int? = nil - ) - async throws -> MLFeatureProvider - { - let audioLength = audioSamples.count - let actualAudioLength = actualLength ?? audioLength // Use provided actual length or default to sample count - - // Create FP32 array first - let audioArrayFP32 = try await sharedMLArrayCache.getArray( - shape: [1, audioLength] as [NSNumber], - dataType: .float32 - ) - - // Copy audio data - audioSamples.withUnsafeBufferPointer { buffer in - let destPtr = audioArrayFP32.dataPointer.bindMemory( - to: Float.self, capacity: audioLength) - memcpy(destPtr, buffer.baseAddress!, audioLength * MemoryLayout.stride) - } - - // Convert to FP16 for Neural Engine - let audioArrayFP16 = try ANEOptimizer.convertToFloat16(audioArrayFP32) - - // Pass the actual audio length, not the padded length - let lengthArray = try createScalarArray(value: actualAudioLength) - - return try createFeatureProvider(features: [ - ("audio_signal", audioArrayFP16), - ("audio_length", lengthArray), - ]) - } - func prepareEncoderInput(_ melspectrogramOutput: MLFeatureProvider) throws -> MLFeatureProvider { // Zero-copy: chain mel-spectrogram outputs directly to encoder inputs if let provider = ZeroCopyFeatureProvider.chain( @@ -379,42 +346,6 @@ public final class AsrManager { logger.info("Decoder state reset for source: \(String(describing: source))") } - internal func transcribeWithState( - _ audioSamples: [Float], decoderState: inout DecoderState - ) - async throws -> ASRResult - { - if config.enableDebug { - logger.debug("transcribeWithState: processing \(audioSamples.count) samples") - // Log decoder state values before processing - let hiddenBefore = ( - decoderState.hiddenState[0].intValue, decoderState.hiddenState[1].intValue - ) - let cellBefore = ( - decoderState.cellState[0].intValue, decoderState.cellState[1].intValue - ) - logger.debug( - "Decoder state before: hidden[\(hiddenBefore.0),\(hiddenBefore.1)], cell[\(cellBefore.0),\(cellBefore.1)]" - ) - } - - let result = try await transcribeUnifiedWithState(audioSamples, decoderState: &decoderState) - - if config.enableDebug { - // Log decoder state values after processing - let hiddenAfter = ( - decoderState.hiddenState[0].intValue, decoderState.hiddenState[1].intValue - ) - let cellAfter = (decoderState.cellState[0].intValue, decoderState.cellState[1].intValue) - logger.debug( - "Decoder state after: hidden[\(hiddenAfter.0),\(hiddenAfter.1)], cell[\(cellAfter.0),\(cellAfter.1)]" - ) - logger.debug("Transcription result: '\(result.text)'") - } - - return result - } - internal func convertTokensWithExistingTimings( _ tokenIds: [Int], timings: [TokenTiming] ) -> ( diff --git a/Sources/FluidAudio/ASR/AsrTranscription.swift b/Sources/FluidAudio/ASR/AsrTranscription.swift index b735acbe..dc8e395f 100644 --- a/Sources/FluidAudio/ASR/AsrTranscription.swift +++ b/Sources/FluidAudio/ASR/AsrTranscription.swift @@ -4,150 +4,58 @@ import OSLog extension AsrManager { - /// Transcribe with FP16 optimization for Neural Engine - public func transcribeWithFP16(_ audioSamples: [Float]) async throws -> ASRResult { + internal func transcribeWithState( + _ audioSamples: [Float], decoderState: inout DecoderState + ) async throws -> ASRResult { guard isAvailable else { throw ASRError.notInitialized } guard audioSamples.count >= 16_000 else { throw ASRError.invalidAudioData } - let startTime = Date() - - if audioSamples.count <= 160_000 { - let originalLength = audioSamples.count - let paddedAudio = padAudioIfNeeded(audioSamples, targetLength: 160_000) - let (tokenIds, encoderSequenceLength) = try await executeMLInferenceWithFP16( - paddedAudio, - originalLength: originalLength, - enableDebug: config.enableDebug + if config.enableDebug { + logger.debug("transcribeWithState: processing \(audioSamples.count) samples") + // Log decoder state values before processing + let hiddenBefore = ( + decoderState.hiddenState[0].intValue, decoderState.hiddenState[1].intValue ) - - return processTranscriptionResult( - tokenIds: tokenIds, - encoderSequenceLength: encoderSequenceLength, - audioSamples: audioSamples, - processingTime: Date().timeIntervalSince(startTime) + let cellBefore = ( + decoderState.cellState[0].intValue, decoderState.cellState[1].intValue + ) + logger.debug( + "Decoder state before: hidden[\(hiddenBefore.0),\(hiddenBefore.1)], cell[\(cellBefore.0),\(cellBefore.1)]" ) } - // For longer audio, use chunking with FP16 - return try await ChunkProcessor( - audioSamples: audioSamples, - chunkSize: 160_000, - enableDebug: config.enableDebug - ).process(using: self, startTime: startTime) - } - - /// Execute ML inference with FP16 optimization - internal func executeMLInferenceWithFP16( - _ paddedAudio: [Float], - originalLength: Int? = nil, - enableDebug: Bool = false - ) async throws -> (tokenIds: [Int], encoderSequenceLength: Int) { - - // Prepare input with ANE-aligned arrays and optionally convert to FP16 - let melspectrogramInput = try await prepareMelSpectrogramInputFP16( - paddedAudio, actualLength: originalLength) - - // Prefetch for ANE if available - if #available(macOS 14.0, iOS 17.0, *), - let audioArray = melspectrogramInput.featureValue(for: "audio_signal")?.multiArrayValue - { - ANEOptimizer.prefetchToNeuralEngine(audioArray) - } - - guard - let melspectrogramOutput = try melspectrogramModel?.prediction( - from: melspectrogramInput, - options: predictionOptions - ) - else { - throw ASRError.processingFailed("Mel-spectrogram model failed") - } - - // Zero-copy encoder input preparation - let encoderInput = try prepareEncoderInput(melspectrogramOutput) - - guard - let encoderOutput = try encoderModel?.prediction( - from: encoderInput, - options: predictionOptions - ) - else { - throw ASRError.processingFailed("Encoder model failed") - } - - let rawEncoderOutput = try extractFeatureValue( - from: encoderOutput, key: "encoder_output", errorMessage: "Invalid encoder output") - let encoderLength = try extractFeatureValue( - from: encoderOutput, key: "encoder_output_length", - errorMessage: "Invalid encoder output length") - - // Encoder output is already optimized for ANE by the model - - let encoderHiddenStates = rawEncoderOutput - let encoderSequenceLength = encoderLength[0].intValue - - var tempDecoderState = try DecoderState() - let tokenIds = try await tdtDecode( - encoderOutput: encoderHiddenStates, - encoderSequenceLength: encoderSequenceLength, - originalAudioSamples: paddedAudio, - decoderState: &tempDecoderState - ) - - return (tokenIds, encoderSequenceLength) - } - - public func transcribeUnified(_ audioSamples: [Float]) async throws -> ASRResult { - guard isAvailable else { throw ASRError.notInitialized } - guard audioSamples.count >= 16_000 else { throw ASRError.invalidAudioData } - let startTime = Date() if audioSamples.count <= 160_000 { let originalLength = audioSamples.count let paddedAudio = padAudioIfNeeded(audioSamples, targetLength: 160_000) let (tokenIds, encoderSequenceLength) = try await executeMLInference( - paddedAudio, originalLength: originalLength, enableDebug: config.enableDebug) - - return processTranscriptionResult( - tokenIds: tokenIds, - encoderSequenceLength: encoderSequenceLength, - audioSamples: audioSamples, - processingTime: Date().timeIntervalSince(startTime) - ) - } - - return try await ChunkProcessor( - audioSamples: audioSamples, - chunkSize: 160_000, - enableDebug: config.enableDebug - ).process(using: self, startTime: startTime) - } - - internal func transcribeUnifiedWithState( - _ audioSamples: [Float], decoderState: inout DecoderState - ) async throws -> ASRResult { - guard isAvailable else { throw ASRError.notInitialized } - guard audioSamples.count >= 16_000 else { throw ASRError.invalidAudioData } - - let startTime = Date() - - if audioSamples.count <= 160_000 { - let originalLength = audioSamples.count - let paddedAudio = padAudioIfNeeded(audioSamples, targetLength: 160_000) - let (tokenIds, encoderSequenceLength) = try await executeMLInferenceWithState( paddedAudio, originalLength: originalLength, enableDebug: config.enableDebug, decoderState: &decoderState ) - return processTranscriptionResult( + let result = processTranscriptionResult( tokenIds: tokenIds, encoderSequenceLength: encoderSequenceLength, audioSamples: audioSamples, processingTime: Date().timeIntervalSince(startTime) ) + + if config.enableDebug { + // Log decoder state values after processing + let hiddenAfter = ( + decoderState.hiddenState[0].intValue, decoderState.hiddenState[1].intValue + ) + let cellAfter = (decoderState.cellState[0].intValue, decoderState.cellState[1].intValue) + logger.debug( + "Decoder state after: hidden[\(hiddenAfter.0),\(hiddenAfter.1)], cell[\(cellAfter.0),\(cellAfter.1)]" + ) + logger.debug("Transcription result: '\(result.text)'") + } + + return result } let result = try await ChunkProcessor( @@ -155,59 +63,12 @@ extension AsrManager { chunkSize: 160_000, enableDebug: config.enableDebug ).process(using: self, startTime: startTime) + + // Note: ChunkProcessor uses its own decoder state, so we don't update the passed-in state return result } internal func executeMLInference( - _ paddedAudio: [Float], - originalLength: Int? = nil, - enableDebug: Bool = false - ) async throws -> (tokenIds: [Int], encoderSequenceLength: Int) { - - let melspectrogramInput = try await prepareMelSpectrogramInput( - paddedAudio, actualLength: originalLength) - - guard - let melspectrogramOutput = try melspectrogramModel?.prediction( - from: melspectrogramInput, - options: predictionOptions - ) - else { - throw ASRError.processingFailed("Mel-spectrogram model failed") - } - - let encoderInput = try prepareEncoderInput(melspectrogramOutput) - guard - let encoderOutput = try encoderModel?.prediction( - from: encoderInput, - options: predictionOptions - ) - else { - throw ASRError.processingFailed("Encoder model failed") - } - - let rawEncoderOutput = try extractFeatureValue( - from: encoderOutput, key: "encoder_output", errorMessage: "Invalid encoder output") - let encoderLength = try extractFeatureValue( - from: encoderOutput, key: "encoder_output_length", - errorMessage: "Invalid encoder output length") - - // Encoder_v2 already outputs in the correct format (B, T, D) - let encoderHiddenStates = rawEncoderOutput - let encoderSequenceLength = encoderLength[0].intValue - - var tempDecoderState = try DecoderState() - let tokenIds = try await tdtDecode( - encoderOutput: encoderHiddenStates, - encoderSequenceLength: encoderSequenceLength, - originalAudioSamples: paddedAudio, - decoderState: &tempDecoderState - ) - - return (tokenIds, encoderSequenceLength) - } - - internal func executeMLInferenceWithState( _ paddedAudio: [Float], originalLength: Int? = nil, enableDebug: Bool = false, @@ -326,8 +187,8 @@ private struct ChunkProcessor { let chunkSamples = Array(audioSamples[position.. ASRResult { // Use optimized transcription with Neural Engine optimizations - let result = try await asrManager.transcribeWithFP16(audioSamples) + let result = try await asrManager.transcribe(audioSamples) if ProcessInfo.processInfo.environment["CI"] != nil && result.text.isEmpty { print("⚠️ CI: Transcription returned empty text") diff --git a/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift b/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift index 4e714274..25b9a0f3 100644 --- a/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift +++ b/Sources/FluidAudioCLI/Commands/DiarizationBenchmark.swift @@ -977,9 +977,6 @@ enum StreamDiarizationBenchmark { let avgFA = results.map { $0.falseAlarmRate }.reduce(0, +) / Float(results.count) let avgSE = results.map { $0.speakerErrorRate }.reduce(0, +) / Float(results.count) let avgRTFx = results.map { $0.rtfx }.reduce(0, +) / Float(results.count) - let avgFragmentation = results.map { $0.speakerFragmentation }.reduce(0, +) / Float(results.count) - let avgLatency90 = results.map { $0.latency90th }.reduce(0, +) / Double(results.count) - let avgLatency99 = results.map { $0.latency99th }.reduce(0, +) / Double(results.count) // Print average row print( diff --git a/Sources/FluidAudioCLI/Utils/AudioProcessor.swift b/Sources/FluidAudioCLI/Utils/AudioProcessor.swift index 2d9f0c48..1f4095cd 100644 --- a/Sources/FluidAudioCLI/Utils/AudioProcessor.swift +++ b/Sources/FluidAudioCLI/Utils/AudioProcessor.swift @@ -7,6 +7,18 @@ struct AudioProcessor { static func loadAudioFile(path: String) async throws -> [Float] { let url = URL(fileURLWithPath: path) + + // Try to load the file directly first + do { + return try await loadAudioFileDirectly(url: url) + } catch { + // If direct loading fails (e.g., FLAC in CI), try converting with ffmpeg + print("Direct audio loading failed, attempting ffmpeg conversion: \(error.localizedDescription)") + return try await loadAudioFileWithFFmpeg(path: path) + } + } + + private static func loadAudioFileDirectly(url: URL) async throws -> [Float] { let audioFile = try AVAudioFile(forReading: url) let format = audioFile.processingFormat @@ -80,6 +92,60 @@ struct AudioProcessor { return resampled } + + /// Load audio file using ffmpeg conversion as fallback for unsupported formats + private static func loadAudioFileWithFFmpeg(path: String) async throws -> [Float] { + let fileManager = FileManager.default + let tempDir = fileManager.temporaryDirectory + let tempWavPath = tempDir.appendingPathComponent("\(UUID().uuidString).wav") + + defer { + // Clean up temp file + try? fileManager.removeItem(at: tempWavPath) + } + + // Convert to WAV using ffmpeg + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/env") + process.arguments = [ + "ffmpeg", + "-i", path, // Input file + "-ar", "16000", // Sample rate + "-ac", "1", // Mono + "-f", "wav", // WAV format + "-y", // Overwrite output + tempWavPath.path, // Output path + "-loglevel", "error", // Only show errors + ] + + let pipe = Pipe() + process.standardError = pipe + + do { + try process.run() + process.waitUntilExit() + + if process.terminationStatus != 0 { + let errorData = pipe.fileHandleForReading.readDataToEndOfFile() + let errorMessage = String(data: errorData, encoding: .utf8) ?? "Unknown error" + throw NSError( + domain: "AudioError", code: 3, + userInfo: [NSLocalizedDescriptionKey: "ffmpeg conversion failed: \(errorMessage)"]) + } + + // Now load the converted WAV file + return try await loadAudioFileDirectly(url: tempWavPath) + + } catch { + // If ffmpeg is not available or fails, throw a more informative error + throw NSError( + domain: "AudioError", code: 4, + userInfo: [ + NSLocalizedDescriptionKey: + "Failed to load audio file. FLAC files require ffmpeg for conversion in CI environment. Error: \(error.localizedDescription)" + ]) + } + } } #endif diff --git a/Tests/FluidAudioTests/AsrManagerTests.swift b/Tests/FluidAudioTests/AsrManagerTests.swift index 08bd220e..2ed7b947 100644 --- a/Tests/FluidAudioTests/AsrManagerTests.swift +++ b/Tests/FluidAudioTests/AsrManagerTests.swift @@ -212,72 +212,6 @@ final class AsrManagerTests: XCTestCase { XCTAssertEqual(length[0].intValue, 100) } - // MARK: - Float16 Inference Tests - - func testPrepareMelSpectrogramInputFP16() async throws { - // Skip this test in CI due to Float16 data type inconsistencies - let isCI = ProcessInfo.processInfo.environment["CI"] != nil - if isCI { - throw XCTSkip("Skipping Float16 test in CI environment") - } - - // Test Float16 input preparation - let audioSamples: [Float] = Array(repeating: 0.1, count: 1000) - let fp16Input = try await manager.prepareMelSpectrogramInputFP16(audioSamples) - - // Verify audio_signal is Float16 - guard let audioSignal = fp16Input.featureValue(for: "audio_signal")?.multiArrayValue else { - XCTFail("Missing audio_signal feature") - return - } - - XCTAssertEqual(audioSignal.shape, [1, 1000] as [NSNumber]) - XCTAssertEqual(audioSignal.dataType, .float16) - - // Verify values are preserved with Float16 precision - for i in 0..