2026 年如何构建 AI 电话呼叫 Agent：分步教程

mkdir ai-phone-agent && cd ai-phone-agent
npm init -y; npm pkg set type="module"
npm install fastify ws dotenv @fastify/formbody @fastify/websocket twilio

OPENAI_API_KEY="sk-..."
# only needed for outbound calls and recording
TWILIO_ACCOUNT_SID="AC..."
TWILIO_API_KEY="SK..."
TWILIO_API_SECRET="..."
TWILIO_NUMBER="+1..."
PUBLIC_HOST="your-tunnel-host.example"   # no scheme, no trailing slash

// index.js  (Fastify + ws). Based on Twilio's "Build an AI Voice Assistant with
// Twilio Voice, OpenAI's Realtime API, and Node.js" tutorial.
// ADAPTED: model name, tools, greeting, barge-in (clear), string concatenation
// instead of template literals, and the booking handlers are ours.
import Fastify from 'fastify';
import WebSocket from 'ws';
import dotenv from 'dotenv';
import fastifyFormBody from '@fastify/formbody';
import fastifyWs from '@fastify/websocket';

dotenv.config();
const { OPENAI_API_KEY } = process.env;
if (!OPENAI_API_KEY) {
    console.error('Missing OPENAI_API_KEY in .env');
    process.exit(1);
}

const MODEL = 'gpt-realtime-2.1';   // ADAPTED (the Twilio tutorial uses 'gpt-realtime')
const VOICE = 'alloy';
const PORT = process.env.PORT || 5050;
const SYSTEM_MESSAGE =
    'You are the phone assistant for Example Dental. Keep answers to one or two short sentences. ' +
    'You can check open appointment slots and book one. Always read back the date, time and name ' +
    'before you call book_appointment. If you are unsure, offer to have a human call back.';

// ADAPTED: toy in-memory calendar, replace with your real scheduling API
const SLOTS = {
    '2026-11-16': ['09:00', '10:30', '14:00'],
    '2026-11-17': ['11:00', '15:30'],
};
const bookings = [];

const TOOLS = [
    {
        type: 'function',
        name: 'check_availability',
        description: 'List open appointment times for a date (YYYY-MM-DD).',
        parameters: {
            type: 'object',
            properties: { date: { type: 'string', description: 'Date as YYYY-MM-DD' } },
            required: ['date'],
        },
    },
    {
        type: 'function',
        name: 'book_appointment',
        description: 'Book one open slot after the caller has confirmed date, time and name.',
        parameters: {
            type: 'object',
            properties: {
                date: { type: 'string', description: 'Date as YYYY-MM-DD' },
                time: { type: 'string', description: 'Time as HH:MM, 24-hour' },
                name: { type: 'string', description: 'Caller full name' },
            },
            required: ['date', 'time', 'name'],
        },
    },
];

function runTool(name, args) {
    if (name === 'check_availability') {
        return { date: args.date, open: SLOTS[args.date] || [] };
    }
    if (name === 'book_appointment') {
        const open = SLOTS[args.date] || [];
        if (!open.includes(args.time)) return { ok: false, error: 'That slot is not available.' };
        SLOTS[args.date] = open.filter((t) => t !== args.time);
        bookings.push({ date: args.date, time: args.time, name: args.name });
        return { ok: true, confirmation: 'BK-' + bookings.length };
    }
    return { ok: false, error: 'Unknown tool ' + name };
}

const LOG_EVENT_TYPES = ['error', 'response.done', 'input_audio_buffer.speech_started', 'session.updated'];

const fastify = Fastify();
fastify.register(fastifyFormBody);
fastify.register(fastifyWs);

fastify.get('/', async (request, reply) => {
    reply.send({ message: 'Twilio Media Stream Server is running!' });
});

// Twilio calls this when the number rings. TwiML: disclose, then open a bidirectional stream.
fastify.all('/incoming-call', async (request, reply) => {
    const twiml =
        '<?xml version="1.0" encoding="UTF-8"?>' +
        '<Response>' +
        '<Say>This call is handled by an AI assistant and may be recorded.</Say>' +   // ADAPTED
        '<Connect><Stream url="wss://' + request.headers.host + '/media-stream" /></Connect>' +
        '</Response>';
    reply.type('text/xml').send(twiml);
});

fastify.register(async (fastify) => {
    fastify.get('/media-stream', { websocket: true }, (connection, req) => {
        console.log('Twilio connected');
        let streamSid = null;
        let greeted = false;

        const openAiWs = new WebSocket('wss://api.openai.com/v1/realtime?model=' + MODEL, {
            headers: { Authorization: 'Bearer ' + OPENAI_API_KEY },
        });

        const sendSessionUpdate = () => {
            openAiWs.send(JSON.stringify({
                type: 'session.update',
                session: {
                    type: 'realtime',
                    model: MODEL,
                    output_modalities: ['audio'],
                    audio: {
                        input: { format: { type: 'audio/pcmu' }, turn_detection: { type: 'server_vad' } },
                        output: { format: { type: 'audio/pcmu' }, voice: VOICE },
                    },
                    instructions: SYSTEM_MESSAGE,
                    tools: TOOLS,            // ADAPTED
                    tool_choice: 'auto',     // ADAPTED
                },
            }));
        };

        openAiWs.on('open', () => setTimeout(sendSessionUpdate, 250));

        openAiWs.on('message', (data) => {
            try {
                const evt = JSON.parse(data);
                if (LOG_EVENT_TYPES.includes(evt.type)) console.log('OpenAI event:', evt.type);

                // ADAPTED: speak first, once, after the session config is applied
                if (evt.type === 'session.updated' && !greeted) {
                    greeted = true;
                    openAiWs.send(JSON.stringify({
                        type: 'response.create',
                        response: { instructions: 'Greet the caller in one sentence and ask how you can help.' },
                    }));
                }

                // model audio -> caller
                if (evt.type === 'response.output_audio.delta' && evt.delta) {
                    connection.send(JSON.stringify({
                        event: 'media',
                        streamSid: streamSid,
                        media: { payload: evt.delta },
                    }));
                }

                // ADAPTED barge-in: caller started talking, flush audio Twilio has buffered
                if (evt.type === 'input_audio_buffer.speech_started' && streamSid) {
                    connection.send(JSON.stringify({ event: 'clear', streamSid: streamSid }));
                }

                // ADAPTED function calling, following OpenAI's Realtime function-calling flow
                if (evt.type === 'response.done') {
                    const output = (evt.response && evt.response.output) || [];
                    for (const item of output) {
                        if (item.type !== 'function_call') continue;
                        const args = JSON.parse(item.arguments || '{}');
                        const result = runTool(item.name, args);
                        openAiWs.send(JSON.stringify({
                            type: 'conversation.item.create',
                            item: {
                                type: 'function_call_output',
                                call_id: item.call_id,
                                output: JSON.stringify(result),
                            },
                        }));
                        openAiWs.send(JSON.stringify({ type: 'response.create' }));
                    }
                }
            } catch (error) {
                console.error('Error processing OpenAI message:', error);
            }
        });

        // caller audio -> model
        connection.on('message', (message) => {
            try {
                const data = JSON.parse(message);
                if (data.event === 'start') {
                    streamSid = data.start.streamSid;
                    console.log('Stream started', streamSid);
                } else if (data.event === 'media') {
                    if (openAiWs.readyState === WebSocket.OPEN) {
                        openAiWs.send(JSON.stringify({ type: 'input_audio_buffer.append', audio: data.media.payload }));
                    }
                }
            } catch (error) {
                console.error('Error parsing Twilio message:', error);
            }
        });

        connection.on('close', () => {
            if (openAiWs.readyState === WebSocket.OPEN) openAiWs.close();
            console.log('Twilio disconnected');
        });
        openAiWs.on('error', (error) => console.error('OpenAI WebSocket error:', error));
    });
});

fastify.listen({ port: PORT, host: '0.0.0.0' }, (err) => {
    if (err) {
        console.error(err);
        process.exit(1);
    }
    console.log('Server is listening on port ' + PORT);
});

node index.js
# 在第二个终端里，用任意隧道暴露 5050 端口，例如：
ngrok http 5050

# 1) platform.openai.com > Project > Webhooks: create a webhook for incoming calls.
# 2) Point your SIP trunk at: sip:[email protected];transport=tls
#    (EU data residency: sip-eu.api.openai.com, India: sip-in.api.openai.com)
# 3) OpenAI POSTs a realtime.call.incoming event to your webhook with data.call_id.
# 4) Accept it (this is the curl from OpenAI's Realtime SIP guide):
curl -X POST "https://api.openai.com/v1/realtime/calls/$CALL_ID/accept" \
  -H "Authorization: Bearer $OPENAI_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
        "type": "realtime",
        "model": "gpt-realtime-2.1",
        "instructions": "You are Alex, a friendly concierge for Example Corp."
      }'
# 5) Attach to the live call for tools and monitoring:
#    wss://api.openai.com/v1/realtime?call_id=$CALL_ID

// 1) You declare tools in session.update (see TOOLS above).
// 2) Caller: "Can I come in Monday the 16th?"  -> model emits, inside response.done:
{
  "type": "response.done",
  "response": {
    "output": [{
      "type": "function_call",
      "name": "check_availability",
      "call_id": "call_abc123",
      "arguments": "{\"date\":\"2026-11-16\"}"
    }]
  }
}
// 3) You run the tool and answer with a function_call_output item, then ask for a response:
{ "type": "conversation.item.create",
  "item": { "type": "function_call_output", "call_id": "call_abc123",
            "output": "{\"date\":\"2026-11-16\",\"open\":[\"09:00\",\"10:30\",\"14:00\"]}" } }
{ "type": "response.create" }
// 4) The model speaks the options to the caller.
// Shapes follow OpenAI's Realtime conversations guide; ids and values are illustrative.

// dial.js  -  place an outbound call that opens the same media stream
// Based on Twilio's Create Call example (API key auth); ADAPTED to inline TwiML with a Stream.
// Run: node dial.js +15551234567
import 'dotenv/config';
import twilio from 'twilio';

const client = twilio(process.env.TWILIO_API_KEY, process.env.TWILIO_API_SECRET, {
    accountSid: process.env.TWILIO_ACCOUNT_SID,
});

const twiml =
    '<Response>' +
    '<Say>Hello, this is an automated assistant from Example Dental calling about your appointment. This call may be recorded.</Say>' +
    '<Connect><Stream url="wss://' + process.env.PUBLIC_HOST + '/media-stream" /></Connect>' +
    '</Response>';

const call = await client.calls.create({
    from: process.env.TWILIO_NUMBER,
    to: process.argv[2],
    twiml: twiml,
});
console.log('Call SID:', call.sid);

// record.js  -  start a dual-channel recording on a live call
// From Twilio's Recordings API docs (example SID and URL are placeholders).
const recording = await client
    .calls('CAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')
    .recordings.create({
        recordingChannels: 'dual',
        recordingStatusCallback: 'https://myapp.com/recording-events',
        recordingStatusCallbackEvent: ['in-progress completed'],
    });

// Vapi: create an assistant (from Vapi's phone quickstart; bun add @vapi-ai/server-sdk)
import { VapiClient } from '@vapi-ai/server-sdk';

const vapi = new VapiClient({ token: process.env.VAPI_API_KEY! });

const assistant = await vapi.assistants.create({
  name: 'Customer Support Assistant',
  model: {
    provider: 'openai',
    model: 'gpt-6-luna',
    reasoningEffort: 'none',
    messages: [{ role: 'system', content: 'You are Alex, a customer service voice assistant for TechSolutions.' }]
  },
  voice: { provider: '11labs', voiceId: 'cgSgspJ2msm6clMCkdW9' },
  firstMessage: 'Hi there, this is Alex from TechSolutions customer support. How can I help you today?'
});

// Your tool server must answer HTTP 200 with this shape, even on failure:
// { "results": [ { "toolCallId": "<id from Vapi>", "result": "<flat string>" } ] }
// On failure use "error" instead of "result".

# Pipecat + Twilio dial-in, commands from Pipecat's Twilio WebSocket guide
ngrok http 7860          # expose the local bot server
uv sync                  # install dependencies
uv run bot.py -t twilio  # run the bot with the Twilio transport