git.giacomolli.uk - onechat/blob - nix/voice-benchmark.nix

   1 { pkgs }:
   2 let
   3   models = pkgs.callPackage ./voice-model.nix { };
   4   english = pkgs.fetchurl {
   5     name = "en-source.flac";
   6     url = "https://archive.org/download/diffusion2023-07-03/diffusion2023-07-03.flac";
   7     hash = "sha256-qXwz8eaFO7A/VNYCmsNyOSYd6EusB4lFMIMDxDDe0lE=";
   8   };
   9   portuguese = pkgs.fetchurl {
  10     name = "pt-source.mp3";
  11     url = "https://archive.org/download/contos_barreto_vcf_librivox/contos_02_barreto_64kb.mp3";
  12     hash = "sha256-k7L2PY8IVL/R1uI/oPqzZsAartmn9bTaA1ivJ80fZHQ=";
  13   };
  14 in
  15 pkgs.writeShellApplication {
  16   name = "voice-benchmark";
  17   runtimeInputs = with pkgs; [
  18     ffmpeg
  19     whisper-cpp
  20     time
  21     gawk
  22   ];
  23   text = ''
  24     if [ "$#" -ne 0 ]; then
  25       echo "Usage: voice-benchmark" >&2
  26       echo "Run eight local CPU transcriptions on pinned public EN/PT samples." >&2
  27       exit 2
  28     fi
  29     work=$(mktemp -d)
  30     trap 'rm -rf "$work"' EXIT
  31     export LC_ALL=C
  32     echo "whisper.cpp ${pkgs.whisper-cpp.version}; CPU only; 8 threads; auto language; no timestamps"
  33     echo "EN: Diffusion Science Radio, Gravity Maverick part 1, CC BY-NC-ND 4.0."
  34     echo "EN sample source: whisper.cpp samples target at v1.9.2."
  35     echo "PT: Lima Barreto, Contos, chapter 2, read by Vicente Costa Filho; LibriVox public domain."
  36     echo "Both excerpts start at 60 seconds. Short: 15 seconds. Long: 300 seconds."
  37     echo "Times include model load and inference, not audio conversion. Peak RSS is per process."
  38     for language in en pt; do
  39       source=${english}
  40       if [ "$language" = pt ]; then source=${portuguese}; fi
  41       for length in 15 300; do
  42         ffmpeg -nostdin -v error -ss 60 -i "$source" -t "$length" \
  43           -ac 1 -ar 48000 -c:a libopus -b:a 32k "$work/$language-$length.ogg"
  44       done
  45     done
  46     echo "Probe: small model on EN Ogg Opus (first second)."
  47     whisper-cli --model ${models.small} --file "$work/en-15.ogg" \
  48       --language auto --no-timestamps --no-gpu --threads 8 --duration 1000 \
  49       --output-txt --output-file "$work/probe" > "$work/probe.stdout" 2> "$work/probe.stderr" || true
  50     cat "$work/probe.stdout" "$work/probe.stderr"
  51     extension=ogg
  52     if [ ! -s "$work/probe.txt" ]; then
  53       extension=wav
  54       echo "Ogg Opus direct: NO. Use ffmpeg to convert to 16 kHz mono PCM WAV."
  55       for language in en pt; do
  56         for length in 15 300; do
  57           ffmpeg -nostdin -v error -i "$work/$language-$length.ogg" \
  58             -ar 16000 -ac 1 -c:a pcm_s16le "$work/$language-$length.wav"
  59         done
  60       done
  61     else
  62       echo "Ogg Opus direct: YES. No WAV conversion required."
  63     fi
  64     printf 'model\tsample\taudio_seconds\twall_seconds\tseconds_per_minute\tpeak_rss_kib\n'
  65     for model in small turbo; do
  66       path=${models.small}
  67       if [ "$model" = turbo ]; then path=${models.turbo}; fi
  68       for language in en pt; do
  69         for length in 15 300; do
  70           sample="$language-$length"
  71           file="$work/$sample.$extension"
  72           duration=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$file")
  73           echo "BEGIN $model $sample: $duration seconds"
  74           ${pkgs.time}/bin/time -f '%e %M' -o "$work/time" \
  75             whisper-cli --model "$path" --file "$file" --language auto \
  76               --no-timestamps --no-gpu --threads 8 --output-txt \
  77               --output-file "$work/transcript" > "$work/stdout" 2> "$work/stderr"
  78           cat "$work/stdout" "$work/stderr"
  79           test -s "$work/transcript.txt"
  80           echo "TRANSCRIPT $model $sample"
  81           cat "$work/transcript.txt"
  82           read -r wall memory < "$work/time"
  83           awk -v model="$model" -v sample="$sample" -v duration="$duration" \
  84             -v wall="$wall" -v memory="$memory" \
  85             'BEGIN {printf "%s\t%s\t%.3f\t%.2f\t%.2f\t%d\n", model, sample, duration, wall, wall*60/duration, memory}'
  86           rm "$work/transcript.txt"
  87           echo "END $model $sample"
  88         done
  89       done
  90     done
  91   '';
  92 }