diff --git a/speak/.gitignore b/speak/.gitignore new file mode 100644 index 0000000..90a13b6 --- /dev/null +++ b/speak/.gitignore @@ -0,0 +1 @@ +/*.bin diff --git a/speak/README.md b/speak/README.md new file mode 100644 index 0000000..80a4dd9 --- /dev/null +++ b/speak/README.md @@ -0,0 +1,55 @@ +# Speak + +Real-time speech-to-text for the acme editor using whisper.cpp streaming. + +## Features + +- Real-time transcription as you speak +- Toggle recording with a single click +- Streams text directly to acme window +- Uses whisper.cpp tiny model for fast inference +- Latency varies by CPU speed (typically a few seconds) + +## Requirements + +- whisper.cpp with stream example built (https://github.com/ggml-org/whisper.cpp) +- Go 1.21+ +- ALSA utils (for audio capture) +- 9p tools for acme integration + +## Installation + +1. Install and build whisper.cpp: +```bash +git clone https://github.com/ggml-org/whisper.cpp +cd whisper.cpp +make stream +# Install whisper-stream to PATH +cp build/bin/whisper-stream ~/bin/ +``` + +2. Install Speak: +```bash +git clone https://github.com/lneely/acme-speak +cd acme-speak +mk install # Downloads tiny model and builds binary to ~/bin/Speak +``` + +## Usage + +From acme, add `Speak` to your window tag and click it: +- First click: Start real-time transcription +- Second click: Stop transcription + +Text appears in the acme window as you speak. + +## Configuration + +Edit `main.go` to change: +- Model: Change `modelPath` constant (tiny, base, small, etc.) +- Latency/accuracy: Adjust `--step` and `--length` parameters + - Current: 500ms step, 5000ms context (good balance) + - Lower latency: 300ms step, 3000ms context (less accurate) + - Higher accuracy: 1000ms step, 8000ms context (more lag) + +Rebuild after changes: `go build -o ~/bin/Speak .` diff --git a/speak/download-coreml-model.sh b/speak/download-coreml-model.sh new file mode 100755 index 0000000..c813736 --- /dev/null +++ b/speak/download-coreml-model.sh @@ -0,0 +1,102 @@ +#!/bin/sh + +printf "whisper.cpp: this script hasn't been maintained and is not functional atm\n" +exit 1 + +# This script downloads Whisper model files that have already been converted to Core ML format. +# This way you don't have to convert them yourself. + +src="https://huggingface.co/datasets/ggerganov/whisper.cpp-coreml" +pfx="resolve/main/ggml" + +# get the path of this script +get_script_path() { + if [ -x "$(command -v realpath)" ]; then + dirname "$(realpath "$0")" + else + _ret="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P)" + echo "$_ret" + fi +} + +script_path="$(get_script_path)" + +# Check if the script is inside a /bin/ directory +case "$script_path" in + */bin) default_download_path="$PWD/models" ;; # Use current directory as default download path if in /bin/ + *) default_download_path="$script_path/models" ;; # Otherwise, use script directory +esac + +models_path="${2:-$default_download_path}" + +# Whisper models +models="tiny.en tiny base.en base small.en small medium.en medium large-v1 large-v2 large-v3 large-v3-turbo" + +# list available models +list_models() { + printf "\n" + printf " Available models:" + for model in $models; do + printf " %s" "$models" + done + printf "\n\n" +} + +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then + printf "Usage: %s [models_path]\n" "$0" + list_models + + exit 1 +fi + +model=$1 + +if ! echo "$models" | grep -q -w "$model"; then + printf "Invalid model: %s\n" "$model" + list_models + + exit 1 +fi + +# download Core ML model + +printf "Downloading Core ML model %s from '%s' ...\n" "$model" "$src" + +cd "$models_path" || exit + +if [ -f "ggml-$model.mlmodel" ]; then + printf "Model %s already exists. Skipping download.\n" "$model" + exit 0 +fi + +if [ -x "$(command -v wget)" ]; then + wget --quiet --show-progress -O ggml-"$model".mlmodel $src/$pfx-"$model".mlmodel +elif [ -x "$(command -v curl)" ]; then + curl -L --output ggml-"$model".mlmodel $src/$pfx-"$model".mlmodel +else + printf "Either wget or curl is required to download models.\n" + exit 1 +fi + + +if [ $? -ne 0 ]; then + printf "Failed to download Core ML model %s \n" "$model" + printf "Please try again later or download the original Whisper model files and convert them yourself.\n" + exit 1 +fi + +# Check if 'whisper-cli' is available in the system PATH +if command -v whisper-cli >/dev/null 2>&1; then + # If found, use 'whisper-cli' (relying on PATH resolution) + whisper_cmd="whisper-cli" +else + # If not found, use the local build version + whisper_cmd="./build/bin/whisper-cli" +fi + +printf "Done! Model '%s' saved in '%s/ggml-%s.bin'\n" "$model" "$models_path" "$model" +printf "Run the following command to compile it:\n\n" +printf " $ xcrun coremlc compile %s/ggml-%s.mlmodel %s\n\n" "$models_path" "$model" "$models_path" +printf "You can now use it like this:\n\n" +printf " $ %s -m %s/ggml-%s.bin -f samples/jfk.wav\n" "$whisper_cmd" "$models_path" "$model" +printf "\n" diff --git a/speak/download-ggml-model.sh b/speak/download-ggml-model.sh new file mode 100755 index 0000000..e554e4d --- /dev/null +++ b/speak/download-ggml-model.sh @@ -0,0 +1,149 @@ +#!/bin/sh + +# This script downloads Whisper model files that have already been converted to ggml format. +# This way you don't have to convert them yourself. + +#src="https://ggml.ggerganov.com" +#pfx="ggml-model-whisper" + +src="https://huggingface.co/ggerganov/whisper.cpp" +pfx="resolve/main/ggml" + +BOLD="\033[1m" +RESET='\033[0m' + +# get the path of this script +get_script_path() { + if [ -x "$(command -v realpath)" ]; then + dirname "$(realpath "$0")" + else + _ret="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P)" + echo "$_ret" + fi +} + +script_path="$(get_script_path)" + +# Check if the script is inside a /bin/ directory +case "$script_path" in + */bin) default_download_path="$PWD/models" ;; # Use current directory as default download path if in /bin/ + *) default_download_path="$script_path/models" ;; # Otherwise, use script directory +esac + +models_path="${2:-$default_download_path}" + +# Whisper models +models="tiny +tiny.en +tiny-q5_1 +tiny.en-q5_1 +tiny-q8_0 +base +base.en +base-q5_1 +base.en-q5_1 +base-q8_0 +small +small.en +small.en-tdrz +small-q5_1 +small.en-q5_1 +small-q8_0 +medium +medium.en +medium-q5_0 +medium.en-q5_0 +medium-q8_0 +large-v1 +large-v2 +large-v2-q5_0 +large-v2-q8_0 +large-v3 +large-v3-q5_0 +large-v3-turbo +large-v3-turbo-q5_0 +large-v3-turbo-q8_0" + +# list available models +list_models() { + printf "\n" + printf "Available models:" + model_class="" + for model in $models; do + this_model_class="${model%%[.-]*}" + if [ "$this_model_class" != "$model_class" ]; then + printf "\n " + model_class=$this_model_class + fi + printf " %s" "$model" + done + printf "\n\n" +} + +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then + printf "Usage: %s [models_path]\n" "$0" + list_models + printf "___________________________________________________________\n" + printf "${BOLD}.en${RESET} = english-only ${BOLD}-q5_[01]${RESET} = quantized ${BOLD}-tdrz${RESET} = tinydiarize\n" + + exit 1 +fi + +model=$1 + +if ! echo "$models" | grep -q -w "$model"; then + printf "Invalid model: %s\n" "$model" + list_models + + exit 1 +fi + +# check if model contains `tdrz` and update the src and pfx accordingly +if echo "$model" | grep -q "tdrz"; then + src="https://huggingface.co/akashmjn/tinydiarize-whisper.cpp" + pfx="resolve/main/ggml" +fi + +echo "$model" | grep -q '^"tdrz"*$' + +# download ggml model + +printf "Downloading ggml model %s from '%s' ...\n" "$model" "$src" + +cd "$models_path" || exit + +if [ -f "ggml-$model.bin" ]; then + printf "Model %s already exists. Skipping download.\n" "$model" + exit 0 +fi + +if [ -x "$(command -v wget2)" ]; then + wget2 --no-config --progress bar -O ggml-"$model".bin $src/$pfx-"$model".bin +elif [ -x "$(command -v curl)" ]; then + curl -L --output ggml-"$model".bin $src/$pfx-"$model".bin +elif [ -x "$(command -v wget)" ]; then + wget --no-config --quiet --show-progress -O ggml-"$model".bin $src/$pfx-"$model".bin +else + printf "Either wget2, curl, or wget is required to download models.\n" + exit 1 +fi + +if [ $? -ne 0 ]; then + printf "Failed to download ggml model %s \n" "$model" + printf "Please try again later or download the original Whisper model files and convert them yourself.\n" + exit 1 +fi + +# Check if 'whisper-cli' is available in the system PATH +if command -v whisper-cli >/dev/null 2>&1; then + # If found, use 'whisper-cli' (relying on PATH resolution) + whisper_cmd="whisper-cli" +else + # If not found, use the local build version + whisper_cmd="./build/bin/whisper-cli" +fi + +printf "Done! Model '%s' saved in '%s/ggml-%s.bin'\n" "$model" "$models_path" "$model" +printf "You can now use it like this:\n\n" +printf " $ %s -m %s/ggml-%s.bin -f samples/jfk.wav\n" "$whisper_cmd" "$models_path" "$model" +printf "\n" diff --git a/speak/download-vad-model.sh b/speak/download-vad-model.sh new file mode 100755 index 0000000..c8dfbd6 --- /dev/null +++ b/speak/download-vad-model.sh @@ -0,0 +1,105 @@ +#!/bin/sh + +# This script downloads Whisper VAD model files that have already been converted +# to ggml format. This way you don't have to convert them yourself. + +src="https://huggingface.co/ggml-org/whisper-vad" +pfx="resolve/main/ggml" + +BOLD="\033[1m" +RESET='\033[0m' + +# get the path of this script +get_script_path() { + if [ -x "$(command -v realpath)" ]; then + dirname "$(realpath "$0")" + else + _ret="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P)" + echo "$_ret" + fi +} + +script_path="$(get_script_path)" + +# Check if the script is inside a /bin/ directory +case "$script_path" in + */bin) default_download_path="$PWD/models" ;; # Use current directory as default download path if in /bin/ + *) default_download_path="$script_path/models" ;; # Otherwise, use script directory +esac + +models_path="${2:-$default_download_path}" + +# Whisper VAD models +models="silero-v5.1.2 silero-v6.2.0" + +# list available models +list_models() { + printf "\n" + printf "Available models:" + model_class="" + for model in $models; do + this_model_class="${model%%[.-]*}" + if [ "$this_model_class" != "$model_class" ]; then + printf "\n " + model_class=$this_model_class + fi + printf " %s" "$model" + done + printf "\n\n" +} + +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then + printf "Usage: %s [models_path]\n" "$0" + list_models + exit 1 +fi + +model=$1 + +if ! echo "$models" | grep -q -w "$model"; then + printf "Invalid model: %s\n" "$model" + list_models + + exit 1 +fi + +# download ggml model +printf "Downloading ggml model %s from '%s' ...\n" "$model" "$src" + +cd "$models_path" || exit + +if [ -f "ggml-$model.bin" ]; then + printf "Model %s already exists. Skipping download.\n" "$model" + exit 0 +fi + +if [ -x "$(command -v wget2)" ]; then + wget2 --no-config --progress bar -O ggml-"$model".bin $src/$pfx-"$model".bin +elif [ -x "$(command -v wget)" ]; then + wget --no-config --quiet --show-progress -O ggml-"$model".bin $src/$pfx-"$model".bin +elif [ -x "$(command -v curl)" ]; then + curl -L --output ggml-"$model".bin $src/$pfx-"$model".bin +else + printf "Either wget or curl is required to download models.\n" + exit 1 +fi + +if [ $? -ne 0 ]; then + printf "Failed to download ggml model %s \n" "$model" + printf "Please try again later or download the original Whisper model files and convert them yourself.\n" + exit 1 +fi + +# Check if 'whisper-cli' is available in the system PATH +if command -v whisper-cli >/dev/null 2>&1; then + # If found, use 'whisper-cli' (relying on PATH resolution) + whisper_cmd="whisper-cli" +else + # If not found, use the local build version + whisper_cmd="./build/bin/whisper-cli" +fi + +printf "Done! Model '%s' saved in '%s/ggml-%s.bin'\n" "$model" "$models_path" "$model" +printf "You can now use it like this:\n\n" +printf " $ %s -vm %s/ggml-%s.bin --vad -f samples/jfk.wav -m models/ggml-base.en.bin\n" "$whisper_cmd" "$models_path" "$model" +printf "\n" diff --git a/speak/go.mod b/speak/go.mod new file mode 100644 index 0000000..a0856a0 --- /dev/null +++ b/speak/go.mod @@ -0,0 +1,5 @@ +module acme-speak + +go 1.21 + +require 9fans.net/go v0.0.5 diff --git a/speak/go.sum b/speak/go.sum new file mode 100644 index 0000000..5089ea2 --- /dev/null +++ b/speak/go.sum @@ -0,0 +1,34 @@ +9fans.net/go v0.0.5 h1:u0H3Et5NowdVm56FDD+aWcDOao9zEo+zolNFsr8qyQk= +9fans.net/go v0.0.5/go.mod h1:Rxvbbc1e+1TyGMjAvLthGTyO97t+6JMQ6ly+Lcs9Uf0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= +golang.org/x/exp v0.0.0-20210405174845-4513512abef3/go.mod h1:I6l2HNBLBZEcrOoCpyKLdY2lHoRZ8lI4x60KMCQDft4= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= +golang.org/x/mobile v0.0.0-20210220033013-bdb1ca9a1e08/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/speak/main.go b/speak/main.go new file mode 100644 index 0000000..d9a4acb --- /dev/null +++ b/speak/main.go @@ -0,0 +1,155 @@ +package main + +import ( + "fmt" + "io" + "log" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "syscall" + + "9fans.net/go/acme" +) + +func readLines(r io.Reader, ch chan string) { + buf := make([]byte, 1) + line := "" + for { + n, err := r.Read(buf) + if n > 0 { + if buf[0] == '\n' { + // Newline = finalized output, send it + if line != "" { + ch <- line + line = "" + } + } else if buf[0] == '\r' { + // Carriage return = overwrite, keep accumulating + line = "" + } else { + line += string(buf[0]) + } + } + if err != nil { + if line != "" { + ch <- line + } + break + } + } +} + +const ( + stateFile = "/tmp/acme-voice-state" + modelPath = "/home/lkn/src/acme-speak/models/ggml-tiny.bin" +) + +func main() { + // Kill any leftover whisper-stream processes + exec.Command("pkill", "-f", "whisper-stream").Run() + + winid := os.Getenv("winid") + + // Check if already running + if pidBytes, err := os.ReadFile(stateFile); err == nil { + // Stop existing stream - use syscall to kill process group + pid := strings.TrimSpace(string(pidBytes)) + pidInt, _ := strconv.Atoi(pid) + if pidInt > 0 { + // Kill the process and its children + exec.Command("pkill", "-9", "-P", pid).Run() + syscall.Kill(pidInt, syscall.SIGKILL) + } + os.Remove(stateFile) + fmt.Fprintln(os.Stderr, "Stopped recording") + return + } + + // Start new stream - optimized for tiny model (faster, lower latency) + cmd := exec.Command("whisper-stream", "-m", modelPath, "-t", "4", "--step", "500", "--length", "5000") + + // Read from both stdout and stderr + stdout, err := cmd.StdoutPipe() + if err != nil { + log.Fatal(err) + } + stderr, err := cmd.StderrPipe() + if err != nil { + log.Fatal(err) + } + + if err := cmd.Start(); err != nil { + log.Fatal(err) + } + + // Save PID + os.WriteFile(stateFile, []byte(strconv.Itoa(cmd.Process.Pid)), 0644) + + // Open acme window if winid is set + var win *acme.Win + if winid != "" { + id, _ := strconv.Atoi(winid) + win, err = acme.Open(id, nil) + if err != nil { + log.Printf("Failed to open acme window: %v", err) + win = nil + } + } + + // Process output from both pipes + ansiRegex := regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`) + + // Merge stdout and stderr + combined := make(chan string, 100) + done := make(chan bool, 2) + + go func() { + readLines(stdout, combined) + done <- true + }() + go func() { + readLines(stderr, combined) + done <- true + }() + + // Close combined when both readers finish + go func() { + <-done + <-done + close(combined) + }() + + lastLine := "" + for line := range combined { + part := ansiRegex.ReplaceAllString(line, "") + part = strings.TrimSpace(part) + + // Filter out debug/empty lines and hallucinations + if part == "" || strings.HasPrefix(part, "whisper_") || + strings.HasPrefix(part, "main:") || strings.HasPrefix(part, "init:") || + part == "[BLANK_AUDIO]" || part == "[Start speaking]" || + strings.HasPrefix(part, "(") || strings.HasPrefix(part, "[Music]") { + continue + } + + // Only write if different from last line + if part != lastLine { + lastLine = part + // Write to acme or stdout + if win != nil { + win.Write("body", []byte(part+" ")) + } else { + fmt.Println(part) + } + } + } + + cmd.Wait() + os.Remove(stateFile) + if win != nil { + win.CloseFiles() + } +} diff --git a/speak/mkfile b/speak/mkfile new file mode 100644 index 0000000..252fb3f --- /dev/null +++ b/speak/mkfile @@ -0,0 +1,6 @@ +install:V: + go build -o $HOME/bin/Speak . + ./download-ggml-model.sh tiny + +clean:V: + rm -f $HOME/bin/Speak diff --git a/speak/models/.gitempty b/speak/models/.gitempty new file mode 100644 index 0000000..e69de29