I am using Gemini TTS 3.1 (although I’ve noticed this issue affects older models as well), and I’ve found that regardless of the settings, the audio generated via Vertex has a slight background noise/hiss during speech. This makes post-editing difficult and degrades the overall quality.
With the exact same settings, a file generated using the standard API does not have this noise. What could be causing this difference?
Files: tts - Google Drive
Here is the code I used to generate the files:
"""
Comparison of the two Gemini TTS generation paths on the SAME text:
- api -> Gemini Developer API (google-genai SDK, endpoint
generativelanguage.googleapis.com), authorized with GEMINI_API_KEY.
This is the path from https://ai.google.dev/gemini-api/docs/speech-generation.
- vertex -> Cloud Text-to-Speech (google-cloud-texttospeech SDK, endpoint
texttospeech.googleapis.com), authorized with a service account
(GOOGLE_APPLICATION_CREDENTIALS) — i.e. how the app works.
Usage:
cd "tools/creator"
source venv/bin/activate
python3 test_tts_compare.py
Files: tools/creator/tts_tests/compare_api.wav and compare_vertex.wav.
Note on the differences between the paths (not a bug, that's how the API works):
- The Developer API controls style through the TEXT (a prefix like "Say warmly: ...").
There is no separate `prompt` field.
- Cloud TTS has a separate `prompt` field (STYLE_PROMPT) next to the text.
Below the style is passed both ways from the same STYLE variable, to keep the
comparison as close as possible.
"""
import os
import time
import wave
from dotenv import load_dotenv
load_dotenv()
# ============================================================
# PARAMETERS
# ============================================================
MODEL = "gemini-3.1-flash-tts-preview" # the same model for both paths
TEXT = (
"Tereska Tomczyk miała szczególny talent - znajdowanie powodów, by robić rzeczy jutro zamiast dziś."
)
# STYLE = the recording direction (Audio Profile + Scene + Director's Notes).
# It goes into the `prompt` field (Cloud TTS) and as a text prefix (Developer API).
# "Scene" is simply a description of the setting and mood — one of the sections below.
# Note: the rigid structure "Scene: The Sound Stage Booth. Director's notes:
# Polish accent" was wrongly blocked by the filter as PROHIBITED_CONTENT
# (a false positive on that combination). A natural sentence goes through fine.
STYLE = (
"You are a warm audiobook narrator with a gentle Polish accent. "
"Read calmly and softly, with tiny dreamy pauses."
)
LANGUAGE = "pl-PL"
VOICE = "Achernar"
# Gemini TTS is natively 24 kHz. SAMPLE_RATE is the target rate of the file:
# - vertex: Cloud TTS resamples server-side,
# - api: we resample the raw 24 kHz PCM locally (audioop),
# so that the file plays at the right pitch/tempo. Anything above 24 kHz is just
# upsampling — it adds no real detail, since the model generates 24 kHz anyway.
NATIVE_RATE = 24000
SAMPLE_RATE = 48000 # target rate of the WAV file
# Cloud TTS (vertex) can generate very hot and clip on loud syllables
# (peaks hitting 0 dBFS -> audible crackle/noise WHILE reading). A negative gain
# gives headroom and eliminates clipping. The Developer API (api) comes out
# quieter and does not need it (and has no such field anyway).
VERTEX_VOLUME_GAIN_DB = -6.0
OUTPUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "tts_tests")
# The Gemini/Vertex safety filter is non-deterministic — the same harmless
# content passes once and gets flagged as PROHIBITED_CONTENT the next time. A few
# retries usually settle it (like the retry ladder in audiobook_generation_service.py).
RETRIES = 3
def _write_wav(path: str, pcm: bytes, rate: int = SAMPLE_RATE):
"""Wraps raw PCM (mono, 16-bit) into a WAV file."""
with wave.open(path, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(2)
wf.setframerate(rate)
wf.writeframes(pcm)
def synth_api() -> str | None:
"""Gemini Developer API via google-genai + GEMINI_API_KEY."""
from google import genai
from google.genai import types
key = os.environ.get("GEMINI_API_KEY")
if not key:
print(" ❌ GEMINI_API_KEY missing in .env")
return None
out_path = os.path.join(OUTPUT_DIR, "compare_api.wav")
print(f"\n=== api === model={MODEL} voice={VOICE} lang={LANGUAGE} "
f"(google-genai / generativelanguage)")
client = genai.Client(api_key=key)
for attempt in range(1, RETRIES + 1):
started = time.time()
try:
resp = client.models.generate_content(
model=MODEL,
contents=f"{STYLE} {TEXT}",
config=types.GenerateContentConfig(
response_modalities=["AUDIO"],
speech_config=types.SpeechConfig(
voice_config=types.VoiceConfig(
prebuilt_voice_config=types.PrebuiltVoiceConfig(
voice_name=VOICE)
)
),
),
)
except Exception as e:
print(f" ❌ API error (api): {str(e).splitlines()[0]}")
return None
# Safe audio extraction: when the filter blocks the content, candidates
# is empty / parts=None — report the reason instead of crashing on [0].
cands = resp.candidates or []
parts = cands[0].content.parts if cands and cands[0].content else None
inline = getattr(parts[0], "inline_data", None) if parts else None
if inline is None:
pf = getattr(resp, "prompt_feedback", None)
reason = getattr(pf, "block_reason", None) or (
cands[0].finish_reason if cands else "no audio")
print(f" ⚠️ attempt {attempt}/{RETRIES}: blocked / no audio "
f"({reason})")
if attempt < RETRIES:
time.sleep(1.5)
continue
print(" ❌ api: failed after retries (safety filter).")
return None
pcm = inline.data
# The model always returns 24 kHz; resample to the target SAMPLE_RATE so
# the WAV header matches the real rate (otherwise the file would play faster).
if SAMPLE_RATE != NATIVE_RATE:
import audioop
pcm, _ = audioop.ratecv(pcm, 2, 1, NATIVE_RATE, SAMPLE_RATE, None)
_write_wav(out_path, pcm, SAMPLE_RATE)
duration = len(pcm) / 2 / SAMPLE_RATE
print(f" ✅ {out_path} ({len(pcm)/1024:.0f} KB, ~{duration:.1f}s audio, "
f"{time.time()-started:.1f}s)")
return out_path
def synth_vertex() -> str | None:
"""Cloud Text-to-Speech via google-cloud-texttospeech + service account."""
from google.cloud import texttospeech
if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"):
print(" ❌ GOOGLE_APPLICATION_CREDENTIALS missing in .env")
return None
out_path = os.path.join(OUTPUT_DIR, "compare_vertex.wav")
print(f"\n=== vertex === model={MODEL} voice={VOICE} lang={LANGUAGE} "
f"(google-cloud-texttospeech / service account)")
client = texttospeech.TextToSpeechClient()
for attempt in range(1, RETRIES + 1):
started = time.time()
try:
resp = client.synthesize_speech(
input=texttospeech.SynthesisInput(text=TEXT, prompt=STYLE),
voice=texttospeech.VoiceSelectionParams(
language_code=LANGUAGE, name=VOICE, model_name=MODEL),
audio_config=texttospeech.AudioConfig(
audio_encoding=texttospeech.AudioEncoding.LINEAR16,
sample_rate_hertz=SAMPLE_RATE,
volume_gain_db=VERTEX_VOLUME_GAIN_DB),
)
except Exception as e:
msg = str(e).splitlines()[0]
# 400 = rejected by the safety filter; retry (it can be random).
is_safety = "usage guidelines" in str(e) or " 400 " in f" {msg} "
if is_safety and attempt < RETRIES:
print(f" ⚠️ attempt {attempt}/{RETRIES}: safety filter "
f"rejected the content, retrying…")
time.sleep(1.5)
continue
print(f" ❌ API error (vertex): {msg}")
return None
with open(out_path, "wb") as f:
f.write(resp.audio_content)
duration = len(resp.audio_content) / 2 / SAMPLE_RATE
print(f" ✅ {out_path} ({len(resp.audio_content)/1024:.0f} KB, "
f"~{duration:.1f}s audio, {time.time()-started:.1f}s)")
return out_path
def main():
os.makedirs(OUTPUT_DIR, exist_ok=True)
print("[1/2] Backend: Gemini Developer API (GEMINI_API_KEY)")
r1 = synth_api()
print("\n[2/2] Backend: Vertex / Cloud TTS (GOOGLE_APPLICATION_CREDENTIALS)")
r2 = synth_vertex()
results = [r for r in (r1, r2) if r]
if results:
print("\nDone. Listen with:")
for path in results:
print(f" afplay '{path}'")
else:
print("\nNothing was generated — check the errors above.")
if __name__ == "__main__":
main()