OpenHuman - TTS / Lip Sync Integration Guide

npm install microsoft-cognitiveservices-speech-sdk

import * as sdk from "microsoft-cognitiveservices-speech-sdk"
import { OpenHuman } from "@openhuman/sdk"
 
const human = new OpenHuman({ canvas })
await human.loadCharacter("./character.ohb")
 
const speechConfig = sdk.SpeechConfig.fromSubscription("YOUR_AZURE_KEY", "YOUR_AZURE_REGION")
speechConfig.speechSynthesisVoiceName = "en-US-JennyNeural"
 
const synthesizer = new sdk.SpeechSynthesizer(speechConfig)

// Collect viseme events before audio starts
const visemeQueue: Array<{ audioOffsetMs: number; visemeId: number }> = []
 
synthesizer.visemeReceived = (s, e) => {
    visemeQueue.push({
        audioOffsetMs: e.audioOffset / 10000, // convert 100ns ticks → ms
        visemeId: e.visemeId,
    })
}
 
synthesizer.speakTextAsync("Hello, how can I help you today?", (result) => {
    // Audio is ready - schedule visemes then play
    const audioData = result.audioData
    playAudioWithLipSync(audioData, visemeQueue)
})

async function playAudioWithLipSync(audioData: ArrayBuffer, visemes: Array<{ audioOffsetMs: number; visemeId: number }>) {
    const ctx = new AudioContext()
    const buffer = await ctx.decodeAudioData(audioData)
    const source = ctx.createBufferSource()
    source.buffer = buffer
    source.connect(ctx.destination)
 
    const startTime = ctx.currentTime
 
    // Schedule each viseme as the audio plays
    visemes.forEach(({ audioOffsetMs, visemeId }) => {
        const triggerAt = startTime + audioOffsetMs / 1000
 
        // Schedule via SDK - handles blend-in/out automatically
        human.scheduleViseme({
            visemeId, // Azure viseme ID 0–21
            triggerAt: triggerAt, // AudioContext time
            audioContext: ctx,
        })
    })
 
    source.start(startTime)
}

async function elevenLabsWithLipSync(text: string, voiceId: string) {
    const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/with-timestamps`, {
        method: "POST",
        headers: {
            "xi-api-key": "YOUR_ELEVENLABS_KEY",
            "Content-Type": "application/json",
        },
        body: JSON.stringify({
            text,
            model_id: "eleven_turbo_v2",
            voice_settings: { stability: 0.5, similarity_boost: 0.75 },
        }),
    })
 
    const data = await response.json()
 
    // data.alignment: { characters[], character_start_times_seconds[], character_end_times_seconds[] }
    // data.audio_base64: base64-encoded MP3
 
    const phonemes = convertAlignmentToPhonemes(data.alignment)
    const audioBuffer = base64ToArrayBuffer(data.audio_base64)
 
    await playAudioWithPhonemes(audioBuffer, phonemes)
}
 
function convertAlignmentToPhonemes(alignment) {
    // ElevenLabs gives character-level timing - map to approximate visemes
    return alignment.characters.map((char, i) => ({
        char,
        startMs: alignment.character_start_times_seconds[i] * 1000,
        endMs: alignment.character_end_times_seconds[i] * 1000,
        viseme: charToViseme(char), // simple character → viseme mapping
    }))
}

const CHAR_TO_VISEME: Record<string, string> = {
    a: "AA",
    e: "EE",
    i: "II",
    o: "OO",
    u: "UU",
    p: "PP",
    b: "PP",
    m: "PP",
    f: "FF",
    v: "FF",
    s: "SS",
    z: "SS",
    r: "RR",
    l: "NN",
    n: "NN",
    k: "KK",
    g: "KK",
    t: "DD",
    d: "DD",
    " ": "sil",
    ".": "sil",
    ",": "sil",
}

import { ParrotSpeechClient } from "@parrotspeech/client"
 
const parrot = new ParrotSpeechClient({ apiKey: "YOUR_KEY" })
 
const result = await parrot.synthesize({
    text: "Hello, how can I help you today?",
    voice: "nova_en_f",
    format: "wav",
    phonemes: true, // request IPA phoneme timestamps
})
 
// result.phonemes: [{ ipa: 'h', startMs: 0, endMs: 60 }, ...]
// result.audioBuffer: ArrayBuffer
 
result.phonemes.forEach(({ ipa, startMs, endMs }) => {
    human.scheduleVisemeByIPA({
        ipa,
        startMs,
        endMs,
        audioContext: ctx,
        audioStartTime: ctx.currentTime,
    })
})
 
playAudio(ctx, result.audioBuffer)

human.scheduleViseme({
  visemeId: number,              // Azure viseme ID (0–21)
  triggerAt: number,             // AudioContext.currentTime (seconds)
  audioContext: AudioContext,
  blendInMs?: number,            // default: 60ms
  blendOutMs?: number,           // default: 80ms
  weight?: number,               // peak weight (default: 1.0)
})

human.scheduleVisemeByName({
  viseme: 'AA' | 'EE' | 'II' | 'OO' | 'UU' | 'PP' | 'FF' |
          'TH' | 'DD' | 'KK' | 'CH' | 'SS' | 'NN' | 'RR' | 'sil',
  triggerAt: number,
  audioContext: AudioContext,
  blendInMs?: number,
  blendOutMs?: number,
  weight?: number,
})

human.scheduleVisemeByIPA({
    ipa: string, // IPA phoneme string
    startMs: number, // ms from audio start
    endMs: number, // ms from audio start
    audioContext: AudioContext,
    audioStartTime: number, // AudioContext time when audio started
})

// Call when interrupting speech (e.g. user interrupts the character)
human.clearVisemeQueue()
human.resetMorphWeights()

human.setVisemeDefaults({
    blendInMs: 50, // faster = more articulate, lower = smoother
    blendOutMs: 70,
    minWeight: 0.0,
    maxWeight: 1.0,
    coarticulationWindow: 2, // how many visemes ahead to pre-blend
})

// Node.js server - using @openhuman/server-sdk
import { OHPSession, MorphFrame } from "@openhuman/server-sdk"
 
async function streamLipSync(session, characterId, text) {
    // 1. Get TTS + phoneme timing
    const tts = await parrot.synthesize({ text, phonemes: true })
 
    // 2. Build morph weight timeline (60fps)
    const timeline = buildMorphTimeline(tts.phonemes, 60)
 
    // 3. Play audio on client
    session.sendAudio(characterId, tts.audioBuffer)
 
    // 4. Stream morph frames in sync
    const startTime = Date.now()
    for (const frame of timeline) {
        const elapsed = Date.now() - startTime
        const delay = frame.timeMs - elapsed
        if (delay > 0) await sleep(delay)
        session.sendMorphFrame(characterId, frame.weights)
    }
}
 
function buildMorphTimeline(phonemes, fps) {
    const frameDuration = 1000 / fps
    const frames = []
 
    phonemes.forEach(({ ipa, startMs, endMs }) => {
        const visemeMorphs = IPA_TO_FACS[ipa] || {}
 
        // Blend in/out across overlapping frames
        for (let t = startMs - 60; t <= endMs + 80; t += frameDuration) {
            const weight = computeVisemeWeight(t, startMs, endMs, 60, 80)
            frames.push({ timeMs: t, weights: scaleWeights(visemeMorphs, weight) })
        }
    })
 
    return mergeAndSortFrames(frames)
}

// Set a happy expression while talking
human.setMorphWeights({
  mouthSmileLeft:  0.6,
  mouthSmileRight: 0.6,
  cheekSquintLeft: 0.3,
  cheekSquintRight: 0.3,
  browInnerUp: 0.2,
})
 
// Lip sync runs on top - jawOpen, mouthFunnel etc. add to the above
human.scheduleViseme({ viseme: 'AA', triggerAt: ..., audioContext: ctx })

human.setMorphBlendMode("additive_clamped") // default - clamp sum to 1.0
human.setMorphBlendMode("additive") // allow overflow (for stylized looks)

human.setHeadMotion({
    enabled: true,
    intensity: 0.4, // 0.0–1.0 - scale of motion
    noddingRate: 0.3, // nods per second during speech
    swayAmplitude: 0.02, // meters - lateral sway
    tiltAmplitude: 1.5, // degrees - head tilt range
})

human.on("lipsync:start", () => {}) // speech begins
human.on("lipsync:viseme", (viseme, weight) => {}) // each viseme fires
human.on("lipsync:end", () => {}) // speech ends
human.on("lipsync:silence", () => {}) // mid-speech silence

human.on("lipsync:end", () => {
    human.setParam("isTalking", false)
    human.animateMorphWeights({}, { duration: 300 }) // smooth reset to neutral
})

import * as sdk from "microsoft-cognitiveservices-speech-sdk"
import { OpenHuman } from "@openhuman/sdk"
 
async function setup() {
    // 1. Init character
    const human = new OpenHuman({ canvas: document.getElementById("canvas") })
    await human.loadCharacter("./character.ohb")
    human.setParam("isTalking", false)
    human.setBlink({ enabled: true })
 
    // 2. Azure TTS setup
    const speechConfig = sdk.SpeechConfig.fromSubscription(KEY, REGION)
    speechConfig.speechSynthesisVoiceName = "en-US-JennyNeural"
    const synthesizer = new sdk.SpeechSynthesizer(speechConfig, null)
 
    // 3. Speak function
    async function speak(text) {
        return new Promise((resolve, reject) => {
            const visemes = []
            const audioCtx = new AudioContext()
 
            synthesizer.visemeReceived = (s, e) => {
                visemes.push({
                    audioOffsetMs: e.audioOffset / 10000,
                    visemeId: e.visemeId,
                })
            }
 
            synthesizer.speakTextAsync(
                text,
                async (result) => {
                    human.setParam("isTalking", true)
 
                    const buffer = await audioCtx.decodeAudioData(result.audioData)
                    const source = audioCtx.createBufferSource()
                    source.buffer = buffer
                    source.connect(audioCtx.destination)
 
                    const startTime = audioCtx.currentTime + 0.1 // 100ms lead time
                    visemes.forEach(({ audioOffsetMs, visemeId }) => {
                        human.scheduleViseme({
                            visemeId,
                            triggerAt: startTime + audioOffsetMs / 1000,
                            audioContext: audioCtx,
                        })
                    })
 
                    source.start(startTime)
                    source.onended = () => {
                        human.setParam("isTalking", false)
                        resolve()
                    }
                },
                reject
            )
        })
    }
 
    return { human, speak }
}
 
// Usage
const { human, speak } = await setup()
await speak("Hello! I am Sarah, your AI assistant.")
await speak("How can I help you today?")