Benchmark local voice transcription

A separate Nix package compares two pinned Whisper models on public
English and Portuguese Ogg Opus excerpts. It prints each transcript,
wall time, and peak memory so the local voice model can be selected
without an external service or private audio. A decode probe selects 16
kHz mono WAV conversion when direct Opus input is not supported.

diff --git a/flake.nix b/flake.nix
index 670dc66643744b28dcd7686b4218d296eef383a2..f20796af642d157318a7e2295cd129e98972f03a 100644 (file)
--- a/flake.nix
+++ b/flake.nix
@@ -30,6 +30,7 @@
         in
         {
           inherit onechat;
         in
         {
           inherit onechat;

+          voice-benchmark = import ./nix/voice-benchmark.nix { inherit pkgs; };

           default = onechat;
         }
       );
           default = onechat;
         }
       );

diff --git a/nix/voice-benchmark.nix b/nix/voice-benchmark.nix
new file mode 100644 (file)
index 0000000..05feae4
--- /dev/null
+++ b/nix/voice-benchmark.nix
@@ -0,0 +1,92 @@
+{ pkgs }:
+let
+  models = pkgs.callPackage ./voice-model.nix { };
+  english = pkgs.fetchurl {
+    name = "en-source.flac";
+    url = "https://archive.org/download/diffusion2023-07-03/diffusion2023-07-03.flac";
+    hash = "sha256-qXwz8eaFO7A/VNYCmsNyOSYd6EusB4lFMIMDxDDe0lE=";
+  };
+  portuguese = pkgs.fetchurl {
+    name = "pt-source.mp3";
+    url = "https://archive.org/download/contos_barreto_vcf_librivox/contos_02_barreto_64kb.mp3";
+    hash = "sha256-k7L2PY8IVL/R1uI/oPqzZsAartmn9bTaA1ivJ80fZHQ=";
+  };
+in
+pkgs.writeShellApplication {
+  name = "voice-benchmark";
+  runtimeInputs = with pkgs; [
+    ffmpeg
+    whisper-cpp
+    time
+    gawk
+  ];
+  text = ''
+    if [ "$#" -ne 0 ]; then
+      echo "Usage: voice-benchmark" >&2
+      echo "Run eight local CPU transcriptions on pinned public EN/PT samples." >&2
+      exit 2
+    fi
+    work=$(mktemp -d)
+    trap 'rm -rf "$work"' EXIT
+    export LC_ALL=C
+    echo "whisper.cpp ${pkgs.whisper-cpp.version}; CPU only; 8 threads; auto language; no timestamps"
+    echo "EN: Diffusion Science Radio, Gravity Maverick part 1, CC BY-NC-ND 4.0."
+    echo "EN sample source: whisper.cpp samples target at v1.9.2."
+    echo "PT: Lima Barreto, Contos, chapter 2, read by Vicente Costa Filho; LibriVox public domain."
+    echo "Both excerpts start at 60 seconds. Short: 15 seconds. Long: 300 seconds."
+    echo "Times include model load and inference, not audio conversion. Peak RSS is per process."
+    for language in en pt; do
+      source=${english}
+      if [ "$language" = pt ]; then source=${portuguese}; fi
+      for length in 15 300; do
+        ffmpeg -nostdin -v error -ss 60 -i "$source" -t "$length" \
+          -ac 1 -ar 48000 -c:a libopus -b:a 32k "$work/$language-$length.ogg"
+      done
+    done
+    echo "Probe: small model on EN Ogg Opus (first second)."
+    whisper-cli --model ${models.small} --file "$work/en-15.ogg" \
+      --language auto --no-timestamps --no-gpu --threads 8 --duration 1000 \
+      --output-txt --output-file "$work/probe" > "$work/probe.stdout" 2> "$work/probe.stderr" || true
+    cat "$work/probe.stdout" "$work/probe.stderr"
+    extension=ogg
+    if [ ! -s "$work/probe.txt" ]; then
+      extension=wav
+      echo "Ogg Opus direct: NO. Use ffmpeg to convert to 16 kHz mono PCM WAV."
+      for language in en pt; do
+        for length in 15 300; do
+          ffmpeg -nostdin -v error -i "$work/$language-$length.ogg" \
+            -ar 16000 -ac 1 -c:a pcm_s16le "$work/$language-$length.wav"
+        done
+      done
+    else
+      echo "Ogg Opus direct: YES. No WAV conversion required."
+    fi
+    printf 'model\tsample\taudio_seconds\twall_seconds\tseconds_per_minute\tpeak_rss_kib\n'
+    for model in small turbo; do
+      path=${models.small}
+      if [ "$model" = turbo ]; then path=${models.turbo}; fi
+      for language in en pt; do
+        for length in 15 300; do
+          sample="$language-$length"
+          file="$work/$sample.$extension"
+          duration=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$file")
+          echo "BEGIN $model $sample: $duration seconds"
+          ${pkgs.time}/bin/time -f '%e %M' -o "$work/time" \
+            whisper-cli --model "$path" --file "$file" --language auto \
+              --no-timestamps --no-gpu --threads 8 --output-txt \
+              --output-file "$work/transcript" > "$work/stdout" 2> "$work/stderr"
+          cat "$work/stdout" "$work/stderr"
+          test -s "$work/transcript.txt"
+          echo "TRANSCRIPT $model $sample"
+          cat "$work/transcript.txt"
+          read -r wall memory < "$work/time"
+          awk -v model="$model" -v sample="$sample" -v duration="$duration" \
+            -v wall="$wall" -v memory="$memory" \
+            'BEGIN {printf "%s\t%s\t%.3f\t%.2f\t%.2f\t%d\n", model, sample, duration, wall, wall*60/duration, memory}'
+          rm "$work/transcript.txt"
+          echo "END $model $sample"
+        done
+      done
+    done
+  '';
+}

diff --git a/nix/voice-model.nix b/nix/voice-model.nix
new file mode 100644 (file)
index 0000000..10ca96f
--- /dev/null
+++ b/nix/voice-model.nix
@@ -0,0 +1,14 @@
+{ fetchurl }:
+let
+  revision = "5359861c739e955e79d9a303bcbc70fb988958b1";
+  model =
+    name: hash:
+    fetchurl {
+      inherit name hash;
+      url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/${revision}/${name}";
+    };
+in
+{
+  small = model "ggml-small.bin" "sha256-G+OpsgY4Z7k35k4ux0gzZKeZF+FX+pjF2UtcH//qmHs=";
+  turbo = model "ggml-large-v3-turbo-q5_0.bin" "sha256-OUIhcJzVrR9AxG5gMcphvOiJMebgiMGIKUxtWlX/p+I=";
+}