feat(crispasr): TTS support (Phase 2) — vibevoice synthesis

backend/go/crispasr/cpp/crispasr_shim.cpp

@@ -23,6 +23,10 @@ const char *crispasr_session_result_segment_text(crispasr_session_result *r,
 int64_t crispasr_session_result_segment_t0(crispasr_session_result *r, int i);
 int64_t crispasr_session_result_segment_t1(crispasr_session_result *r, int i);
 void crispasr_session_result_free(crispasr_session_result *r);
+float *crispasr_session_synthesize(crispasr_session *s, const char *text,
+                                   int *out_n_samples);
+void crispasr_pcm_free(float *pcm);
+int crispasr_session_set_speaker_name(crispasr_session *s, const char *name);
 }
 
 static crispasr_session *g_session = nullptr;
@@ -199,3 +203,21 @@ int64_t get_segment_t1(int i) {
 const char *get_backend(void) {
   return g_session ? crispasr_session_backend(g_session) : "";
 }
+
+// TTS uses the already-open session (crispasr_session_open auto-detects a TTS
+// model). Output is 24 kHz mono float PCM (upstream CrispASR convention),
+// malloc'd by the C API; the caller must release it via tts_free.
+float *tts_synthesize(const char *text, int *out_n_samples) {
+  if (out_n_samples) *out_n_samples = 0;
+  if (!g_session || !text) return nullptr;
+  return crispasr_session_synthesize(g_session, text, out_n_samples);
+}
+
+void tts_free(float *pcm) {
+  if (pcm) crispasr_pcm_free(pcm);
+}
+
+int tts_set_voice(const char *name) {
+  if (!g_session || !name || !*name) return 0;
+  return crispasr_session_set_speaker_name(g_session, name);
+}

backend/go/crispasr/cpp/crispasr_shim.h

@@ -14,4 +14,7 @@ int64_t get_segment_t0(int i);
 int64_t get_segment_t1(int i);
 const char *get_backend(void);
 void set_abort(int v);
+float *tts_synthesize(const char *text, int *out_n_samples); // 24kHz mono float, malloc'd; NULL on failure
+void tts_free(float *pcm);
+int tts_set_voice(const char *name); // best-effort speaker selection; 0 ok
 }

backend/go/crispasr/gocrispasr.go

@@ -9,6 +9,7 @@
 	"sync"
 	"unsafe"
 
+	"github.com/go-audio/audio"
 	"github.com/go-audio/wav"
 	"github.com/mudler/LocalAI/pkg/grpc/base"
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
@@ -27,6 +28,9 @@
 	CppGetSegmentEnd   func(i int) int64
 	CppGetBackend      func() string
 	CppSetAbort        func(v int)
+	CppTTSSynthesize   func(text string, outNSamples unsafe.Pointer) uintptr
+	CppTTSFree         func(ptr uintptr)
+	CppTTSSetVoice     func(name string) int
 )
 
 type CrispASR struct {
@@ -80,7 +84,7 @@
 
 	// unsafeptr warning is caused by segsPtr being on the stack and therefor being subject to stack copying AFAICT
 	// however the stack shouldn't have grown between setting segsPtr and now, also the memory pointed to is allocated by C++
 	segs := unsafe.Slice((*float32)(unsafe.Pointer(segsPtr)), segsLen)
 
 	vadSegments := []*pb.VADSegment{}
 	for i := range len(segs) >> 1 {
@@ -106,7 +110,7 @@
 	if err != nil {
 		return pb.TranscriptResult{}, err
 	}
 	defer os.RemoveAll(dir)
 
 	convertedPath := filepath.Join(dir, "converted.wav")
 
@@ -118,7 +122,7 @@
 	if err != nil {
 		return pb.TranscriptResult{}, err
 	}
 	defer fh.Close()
 
 	d := wav.NewDecoder(fh)
 	buf, err := d.FullPCMBuffer()
@@ -304,3 +308,121 @@
 	results <- &pb.TranscriptStreamResponse{FinalResult: final}
 	return nil
 }
+
+// synthesize returns 24 kHz mono float32 PCM for text via the open session.
+func (w *CrispASR) synthesize(text string) ([]float32, error) {
+	if text == "" {
+		return nil, fmt.Errorf("crispasr: TTS requires non-empty text")
+	}
+	var n int32
+	ptr := CppTTSSynthesize(text, unsafe.Pointer(&n))
+	if ptr == 0 || n <= 0 {
+		return nil, fmt.Errorf("crispasr: synthesis failed (the loaded model may not be a supported TTS backend, or needs extra config e.g. orpheus SNAC codec)")
+	}
+	defer CppTTSFree(ptr)
+	src := unsafe.Slice((*float32)(unsafe.Pointer(ptr)), int(n))
+	out := make([]float32, int(n)) // copy out of C memory before free
+	copy(out, src)
+	return out, nil
+}
+
+// setVoice applies a per-call speaker/voice override (best effort). CrispASR
+// returns a negative code when the active backend can't honor the name; we log
+// it rather than fail, so an unknown voice falls back to the default speaker.
+func setVoice(voice string) {
+	v := strings.TrimSpace(voice)
+	if v == "" {
+		return
+	}
+	if rc := CppTTSSetVoice(v); rc != 0 {
+		fmt.Fprintf(os.Stderr, "crispasr: voice %q not applied by the active TTS backend (rc=%d); using default\n", v, rc)
+	}
+}
+
+func (w *CrispASR) TTS(req *pb.TTSRequest) error {
+	if req.Dst == "" {
+		return fmt.Errorf("crispasr: TTS requires a destination path")
+	}
+	setVoice(req.Voice)
+	pcm, err := w.synthesize(req.Text)
+	if err != nil {
+		return err
+	}
+	return writeWAV24k(req.Dst, pcm)
+}
+
+// TTSStream is the streaming counterpart to TTS. CrispASR has no progressive
+// (native streaming) synth, so we synthesize the whole utterance, encode it to
+// a 24 kHz WAV, and emit the encoded bytes as a single chunk. The gRPC server
+// wrapper (pkg/grpc/server.go:TTSStream) ranges over the channel until it is
+// closed, so this method owns the close - mirrors vibevoice-cpp's TTSStream.
+func (w *CrispASR) TTSStream(req *pb.TTSRequest, results chan []byte) error {
+	defer close(results)
+
+	if req.Text == "" {
+		return fmt.Errorf("crispasr: TTSStream requires text")
+	}
+	setVoice(req.Voice)
+	pcm, err := w.synthesize(req.Text)
+	if err != nil {
+		return err
+	}
+
+	tmp, err := os.CreateTemp("", "crispasr-tts-stream-*.wav")
+	if err != nil {
+		return fmt.Errorf("crispasr: tempfile: %w", err)
+	}
+	dst := tmp.Name()
+	if err := tmp.Close(); err != nil {
+		return fmt.Errorf("crispasr: close tempfile: %w", err)
+	}
+	defer func() { _ = os.Remove(dst) }()
+
+	if err := writeWAV24k(dst, pcm); err != nil {
+		return err
+	}
+
+	encoded, err := os.ReadFile(dst)
+	if err != nil {
+		return fmt.Errorf("crispasr: read tempfile: %w", err)
+	}
+	results <- encoded
+	return nil
+}
+
+// writeWAV24k writes pcm as a 24000 Hz, mono, 16-bit PCM WAV at dst.
+func writeWAV24k(dst string, pcm []float32) error {
+	f, err := os.Create(dst)
+	if err != nil {
+		return fmt.Errorf("crispasr: create %q: %w", dst, err)
+	}
+
+	enc := wav.NewEncoder(f, 24000, 16, 1, 1)
+	ints := make([]int, len(pcm))
+	for i, s := range pcm {
+		if s > 1 {
+			s = 1
+		} else if s < -1 {
+			s = -1
+		}
+		ints[i] = int(s * 32767)
+	}
+	buf := &audio.IntBuffer{
+		Format:         &audio.Format{NumChannels: 1, SampleRate: 24000},
+		Data:           ints,
+		SourceBitDepth: 16,
+	}
+	if err := enc.Write(buf); err != nil {
+		_ = enc.Close()
+		_ = f.Close()
+		return fmt.Errorf("crispasr: encode WAV: %w", err)
+	}
+	if err := enc.Close(); err != nil {
+		_ = f.Close()
+		return fmt.Errorf("crispasr: finalize WAV: %w", err)
+	}
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("crispasr: close %q: %w", dst, err)
+	}
+	return nil
+}

backend/go/crispasr/gocrispasr_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"testing"
@@ -50,6 +51,9 @@ func ensureLibLoaded() {
 		purego.RegisterLibFunc(&CppGetSegmentEnd, gosd, "get_segment_t1")
 		purego.RegisterLibFunc(&CppGetBackend, gosd, "get_backend")
 		purego.RegisterLibFunc(&CppSetAbort, gosd, "set_abort")
+		purego.RegisterLibFunc(&CppTTSSynthesize, gosd, "tts_synthesize")
+		purego.RegisterLibFunc(&CppTTSFree, gosd, "tts_free")
+		purego.RegisterLibFunc(&CppTTSSetVoice, gosd, "tts_set_voice")
 	})
 	if libLoadErr != nil {
 		Skip("whisper library not loadable: " + libLoadErr.Error())
@@ -68,6 +72,17 @@ func fixturesOrSkip() (string, string) {
 	return modelPath, audioPath
 }
 
+// ttsModelOrSkip returns the TTS model path or skips the spec when the env var
+// is unset. Like the transcription fixtures, this never runs in default CI — it
+// needs a real TTS model (e.g. a vibevoice GGUF) on disk.
+func ttsModelOrSkip() string {
+	modelPath := os.Getenv("CRISPASR_TTS_MODEL_PATH")
+	if modelPath == "" {
+		Skip("set CRISPASR_TTS_MODEL_PATH to run this spec")
+	}
+	return modelPath
+}
+
 var _ = Describe("CrispASR", func() {
 	Context("AudioTranscription cancellation", func() {
 		It("returns codes.Canceled on a pre-cancelled context and still succeeds afterwards", func() {
@@ -153,4 +168,24 @@ var _ = Describe("CrispASR", func() {
 				"concat(deltas) must equal final.Text")
 		})
 	})
+
+	Context("TTS", func() {
+		It("synthesizes a non-empty WAV", func() {
+			ttsModel := ttsModelOrSkip()
+			ensureLibLoaded()
+
+			w := &CrispASR{}
+			Expect(w.Load(&pb.ModelOptions{ModelFile: ttsModel})).To(Succeed())
+
+			dst := filepath.Join(GinkgoT().TempDir(), "out.wav")
+			Expect(w.TTS(&pb.TTSRequest{Text: "Hello from CrispASR.", Dst: dst})).To(Succeed())
+
+			info, err := os.Stat(dst)
+			Expect(err).ToNot(HaveOccurred(), "synthesized WAV should exist at %q", dst)
+			// A real 24 kHz mono WAV is a 44-byte header plus samples; anything
+			// this small would mean an empty/failed synth.
+			Expect(info.Size()).To(BeNumerically(">", 1024),
+				"expected a non-trivial WAV, got %d bytes", info.Size())
+		})
+	})
 })

backend/go/crispasr/main.go

@@ -39,6 +39,9 @@ func main() {
 		{&CppGetSegmentEnd, "get_segment_t1"},
 		{&CppGetBackend, "get_backend"},
 		{&CppSetAbort, "set_abort"},
+		{&CppTTSSynthesize, "tts_synthesize"},
+		{&CppTTSFree, "tts_free"},
+		{&CppTTSSetVoice, "tts_set_voice"},
 	}
 
 	for _, lf := range libFuncs {

gallery/index.yaml

@@ -32256,3 +32256,24 @@
   files:
     - filename: vibevoice-asr-q4_k.gguf
       uri: huggingface://cstr/vibevoice-asr-GGUF/vibevoice-asr-q4_k.gguf
+- name: vibevoice-tts-crispasr
+  url: github:mudler/LocalAI/gallery/vibevoice-tts-crispasr.yaml@master
+  urls:
+    - https://huggingface.co/cstr/vibevoice-realtime-0.5b-GGUF
+  description: |
+    VibeVoice Realtime 0.5B text-to-speech (TTS) model, synthesized through the CrispASR backend. Produces 24 kHz mono audio; runs end-to-end on CPU with a built-in default voice. Default GGUF size ~636 MB.
+  tags:
+    - crispasr
+    - tts
+    - text-to-speech
+    - gguf
+  overrides:
+    backend: crispasr
+    known_usecases:
+      - tts
+    name: vibevoice-tts-crispasr
+    parameters:
+      model: vibevoice-realtime-0.5b-q4_k.gguf
+  files:
+    - filename: vibevoice-realtime-0.5b-q4_k.gguf
+      uri: huggingface://cstr/vibevoice-realtime-0.5b-GGUF/vibevoice-realtime-0.5b-q4_k.gguf

gallery/vibevoice-tts-crispasr.yaml

@@ -0,0 +1,7 @@
+---
+name: "vibevoice-tts-crispasr"
+
+config_file: |
+  backend: crispasr
+  parameters:
+    model: vibevoice-realtime-0.5b-q4_k.gguf