{ pkgs }: let models = pkgs.callPackage ./voice-model.nix { }; english = pkgs.fetchurl { name = "en-source.flac"; url = "https://archive.org/download/diffusion2023-07-03/diffusion2023-07-03.flac"; hash = "sha256-qXwz8eaFO7A/VNYCmsNyOSYd6EusB4lFMIMDxDDe0lE="; }; portuguese = pkgs.fetchurl { name = "pt-source.mp3"; url = "https://archive.org/download/contos_barreto_vcf_librivox/contos_02_barreto_64kb.mp3"; hash = "sha256-k7L2PY8IVL/R1uI/oPqzZsAartmn9bTaA1ivJ80fZHQ="; }; in pkgs.writeShellApplication { name = "voice-benchmark"; runtimeInputs = with pkgs; [ ffmpeg whisper-cpp time gawk ]; text = '' if [ "$#" -ne 0 ]; then echo "Usage: voice-benchmark" >&2 echo "Run eight local CPU transcriptions on pinned public EN/PT samples." >&2 exit 2 fi work=$(mktemp -d) trap 'rm -rf "$work"' EXIT export LC_ALL=C echo "whisper.cpp ${pkgs.whisper-cpp.version}; CPU only; 8 threads; auto language; no timestamps" echo "EN: Diffusion Science Radio, Gravity Maverick part 1, CC BY-NC-ND 4.0." echo "EN sample source: whisper.cpp samples target at v1.9.2." echo "PT: Lima Barreto, Contos, chapter 2, read by Vicente Costa Filho; LibriVox public domain." echo "Both excerpts start at 60 seconds. Short: 15 seconds. Long: 300 seconds." echo "Times include model load and inference, not audio conversion. Peak RSS is per process." for language in en pt; do source=${english} if [ "$language" = pt ]; then source=${portuguese}; fi for length in 15 300; do ffmpeg -nostdin -v error -ss 60 -i "$source" -t "$length" \ -ac 1 -ar 48000 -c:a libopus -b:a 32k "$work/$language-$length.ogg" done done echo "Probe: small model on EN Ogg Opus (first second)." whisper-cli --model ${models.small} --file "$work/en-15.ogg" \ --language auto --no-timestamps --no-gpu --threads 8 --duration 1000 \ --output-txt --output-file "$work/probe" > "$work/probe.stdout" 2> "$work/probe.stderr" || true cat "$work/probe.stdout" "$work/probe.stderr" extension=ogg if [ ! -s "$work/probe.txt" ]; then extension=wav echo "Ogg Opus direct: NO. Use ffmpeg to convert to 16 kHz mono PCM WAV." for language in en pt; do for length in 15 300; do ffmpeg -nostdin -v error -i "$work/$language-$length.ogg" \ -ar 16000 -ac 1 -c:a pcm_s16le "$work/$language-$length.wav" done done else echo "Ogg Opus direct: YES. No WAV conversion required." fi printf 'model\tsample\taudio_seconds\twall_seconds\tseconds_per_minute\tpeak_rss_kib\n' for model in small turbo; do path=${models.small} if [ "$model" = turbo ]; then path=${models.turbo}; fi for language in en pt; do for length in 15 300; do sample="$language-$length" file="$work/$sample.$extension" duration=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$file") echo "BEGIN $model $sample: $duration seconds" ${pkgs.time}/bin/time -f '%e %M' -o "$work/time" \ whisper-cli --model "$path" --file "$file" --language auto \ --no-timestamps --no-gpu --threads 8 --output-txt \ --output-file "$work/transcript" > "$work/stdout" 2> "$work/stderr" cat "$work/stdout" "$work/stderr" test -s "$work/transcript.txt" echo "TRANSCRIPT $model $sample" cat "$work/transcript.txt" read -r wall memory < "$work/time" awk -v model="$model" -v sample="$sample" -v duration="$duration" \ -v wall="$wall" -v memory="$memory" \ 'BEGIN {printf "%s\t%s\t%.3f\t%.2f\t%.2f\t%d\n", model, sample, duration, wall, wall*60/duration, memory}' rm "$work/transcript.txt" echo "END $model $sample" done done done ''; }