Files
FluidAudio/Sources/FluidAudioCLI/Commands/DownloadCommand.swift
Alex 2593f55415 Add Japanese ASR support with JSUT and Common Voice datasets (#478)
## Summary

Adds comprehensive Japanese ASR support to FluidAudio with benchmark
datasets and CLI commands.

## Changes

### Core Japanese ASR Support
- **CtcJaManager.swift** - Japanese CTC transcription manager
(actor-based)
- **CtcJaModels.swift** - Japanese model loading and management
- **ModelNames.swift** - Added Japanese model registry (`parakeetCtcJa`,
`CTCJa` enum)
- **AsrModels.swift** - Added `.ctcJa` model version (3,072 vocab, 1,024
hidden, blank_id=3072)
- **AsrManager.swift** - Added `.ctcJa` case with error directing to
`CtcJaManager`

### CLI Commands
- **JapaneseAsrBenchmark.swift** (459 lines) - New `ja-benchmark`
command
  - JSUT basic5000 dataset support
  - Mozilla Common Voice (MCV) test set support
  - Auto-download capability
  - CER (Character Error Rate) evaluation
- **DownloadCommand.swift** - Added JSUT and MCV Japanese dataset
downloads
- **TranscribeCommand.swift** - Added `.ctcJa` model version support
- **AsrBenchmark.swift** - Added `.ctcJa` switch case

### Dataset Support
- **JapaneseDatasetDownloader.swift** (387 lines) - Dataset download and
parsing
  - JSUT basic5000 (5,000 sentences, clean studio recordings)
  - Mozilla Common Voice Japanese test split
  - Efficient streaming downloads
  - Metadata extraction and validation

## Usage

### CLI Commands
```bash
# Benchmark on JSUT basic5000 (100 samples)
swift run fluidaudiocli ja-benchmark --dataset jsut --samples 100

# Benchmark on Common Voice test (500 samples, auto-download)
swift run fluidaudiocli ja-benchmark --dataset cv-test --samples 500 --auto-download

# Download datasets
swift run fluidaudiocli download --dataset jsut
swift run fluidaudiocli download --dataset cv-ja-test
```

### Swift API
```swift
// Load and use Japanese CTC transcription
let manager = try await CtcJaManager.load()
let text = try manager.transcribe(audioURL: japaneseAudioFile)
```

## Model Info
- **Repo**: `FluidInference/parakeet-ctc-0.6b-ja-coreml`
- **Architecture**: 600M parameter CTC-only
- **Vocabulary**: 3,072 Japanese SentencePiece tokens + 1 blank (id:
3072)
- **Encoder**: 1,024 hidden size
- **Expected CER**: 6.5% on JSUT basic5000, 13.3% on MCV 16.1 test

## Testing
- ✅ Builds successfully (`swift build`)
- ✅ Model loading integration tested
- ✅ CLI commands compile and link correctly
- ⏳ Runtime benchmark testing pending (requires model download)

## Related
- Mobius PR #39: Japanese CTC CoreML conversion
(https://github.com/FluidInference/mobius/pull/39)

🤖 Generated with Claude Code
<!-- devin-review-badge-begin -->

---

<a href="https://app.devin.ai/review/fluidinference/fluidaudio/pull/478"
target="_blank">
  <picture>
<source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-open-in-devin-review-dark.svg?v=1">
<img
src="https://static.devin.ai/assets/gh-open-in-devin-review-light.svg?v=1"
alt="Open with Devin">
  </picture>
</a>
<!-- devin-review-badge-end -->

---------
2026-04-04 12:57:32 -04:00

125 lines
5.1 KiB
Swift

#if os(macOS)
import FluidAudio
import Foundation
/// Handler for the 'download' command - downloads benchmark datasets
enum DownloadCommand {
private static let logger = AppLogger(category: "Download")
static func run(arguments: [String]) async {
var dataset = "all"
var forceDownload = false
// Parse arguments
var i = 0
while i < arguments.count {
switch arguments[i] {
case "--dataset":
if i + 1 < arguments.count {
dataset = arguments[i + 1]
i += 1
}
case "--force":
forceDownload = true
default:
logger.warning("Unknown option: \(arguments[i])")
}
i += 1
}
logger.info("📥 Starting dataset download")
logger.info(" Dataset: \(dataset)")
logger.info(" Force download: \(forceDownload ? "enabled" : "disabled")")
switch dataset.lowercased() {
case "ami-sdm":
await DatasetDownloader.downloadAMIDataset(variant: .sdm, force: forceDownload)
case "ami-ihm":
await DatasetDownloader.downloadAMIDataset(variant: .ihm, force: forceDownload)
case "ami-annotations":
await DatasetDownloader.downloadAMIAnnotations(force: forceDownload)
case "vad":
// Default to mini100 for more test data
await DatasetDownloader.downloadVadDataset(
force: forceDownload, dataset: "mini100")
case "vad-mini50":
await DatasetDownloader.downloadVadDataset(force: forceDownload, dataset: "mini50")
case "vad-mini100":
await DatasetDownloader.downloadVadDataset(force: forceDownload, dataset: "mini100")
case "musan-full":
await DatasetDownloader.downloadFullMusanDataset(force: forceDownload)
case "voices-subset":
await DatasetDownloader.downloadVoicesSubset(force: forceDownload)
case "librispeech-test-clean":
let benchmark = ASRBenchmark()
do {
try await benchmark.downloadLibriSpeech(
subset: "test-clean", forceDownload: forceDownload)
} catch {
logger.error("Failed to download LibriSpeech test-clean: \(error)")
exit(1)
}
case "librispeech-test-other":
let benchmark = ASRBenchmark()
do {
try await benchmark.downloadLibriSpeech(
subset: "test-other", forceDownload: forceDownload)
} catch {
logger.error("Failed to download LibriSpeech test-other: \(error)")
exit(1)
}
case "earnings22-kws":
await DatasetDownloader.downloadEarnings22KWS(force: forceDownload)
case "jsut-basic5000":
await DatasetDownloader.downloadJSUTBasic5000(force: forceDownload)
case "cv-corpus-ja-test":
await DatasetDownloader.downloadCommonVoiceJapanese(
force: forceDownload, split: .test)
case "all":
await DatasetDownloader.downloadAMIDataset(variant: .sdm, force: forceDownload)
await DatasetDownloader.downloadAMIDataset(variant: .ihm, force: forceDownload)
await DatasetDownloader.downloadVadDataset(force: forceDownload, dataset: "mini100")
default:
logger.error("Unsupported dataset: \(dataset)")
printUsage()
exit(1)
}
}
private static func printUsage() {
logger.info(
"""
Download Command Usage:
fluidaudio download [options]
Options:
--dataset <name> Dataset to download (default: all)
--force Force re-download even if exists
Available datasets:
ami-sdm AMI SDM subset
ami-ihm AMI IHM subset
ami-annotations AMI annotation files
vad, vad-mini50, VAD evaluation datasets
vad-mini100
musan-full Full MUSAN dataset (~109 hours)
voices-subset VOiCES small subset (clean/noisy pairs)
librispeech-test-clean LibriSpeech test-clean subset
librispeech-test-other LibriSpeech test-other subset
earnings22-kws Earnings22 keyword spotting dataset
jsut-basic5000 JSUT Japanese speech dataset (5k utts)
cv-corpus-ja-test Common Voice Japanese test split
parakeet-models Parakeet ASR models
all All diarization datasets
Examples:
fluidaudio download --dataset ami-sdm
fluidaudio download --dataset librispeech-test-clean --force
fluidaudio download --dataset jsut-basic5000
fluidaudio download --dataset cv-corpus-ja-test
"""
)
}
}
#endif