kent
5d0f2d1346
레거시 5130.sam.kr/ai_sam의 Google Gemini Live API 음성 어시스턴트를 MNG 프로젝트로 이전 (React → Pure JS + Blade) 변경 내용: - GeminiController: API 키 제공 엔드포인트 추가 - sam-ai-live.js: LiveManager, AudioVisualizer ES 모듈 - sam-ai-menu.blade.php: 전면 재작성 (Tailwind UI) - 환경변수: GEMINI_API_KEY, GEMINI_PROJECT_ID 추가 기능: - 실시간 음성 입출력 (WebAudio API) - UI 도구: navigateToPage, searchDocuments - 오디오 시각화 (Canvas API) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
606 lines
22 KiB
JavaScript
606 lines
22 KiB
JavaScript
/**
|
|
* SAM AI Live - Google Gemini Live API 음성 어시스턴트
|
|
*
|
|
* 레거시 5130.sam.kr/ai_sam 기능을 MNG로 이전
|
|
* - 음성 입출력 (WebAudio API)
|
|
* - Google Gemini Live API 연동
|
|
* - UI 제어 도구 (메뉴 이동, 파일 검색)
|
|
*/
|
|
|
|
// ============================================================================
|
|
// Audio Utilities
|
|
// ============================================================================
|
|
|
|
/**
|
|
* Base64 문자열을 Uint8Array로 디코딩
|
|
*/
|
|
export function decodeBase64(base64) {
|
|
const binaryString = atob(base64);
|
|
const len = binaryString.length;
|
|
const bytes = new Uint8Array(len);
|
|
for (let i = 0; i < len; i++) {
|
|
bytes[i] = binaryString.charCodeAt(i);
|
|
}
|
|
return bytes;
|
|
}
|
|
|
|
/**
|
|
* Uint8Array를 Base64 문자열로 인코딩
|
|
*/
|
|
export function encodeBase64(bytes) {
|
|
let binary = '';
|
|
const len = bytes.byteLength;
|
|
for (let i = 0; i < len; i++) {
|
|
binary += String.fromCharCode(bytes[i]);
|
|
}
|
|
return btoa(binary);
|
|
}
|
|
|
|
/**
|
|
* PCM 데이터를 AudioBuffer로 변환
|
|
*/
|
|
export async function decodeAudioData(data, ctx, sampleRate = 24000, numChannels = 1) {
|
|
const dataInt16 = new Int16Array(data.buffer);
|
|
const frameCount = dataInt16.length / numChannels;
|
|
const buffer = ctx.createBuffer(numChannels, frameCount, sampleRate);
|
|
|
|
for (let channel = 0; channel < numChannels; channel++) {
|
|
const channelData = buffer.getChannelData(channel);
|
|
for (let i = 0; i < frameCount; i++) {
|
|
channelData[i] = dataInt16[i * numChannels + channel] / 32768.0;
|
|
}
|
|
}
|
|
return buffer;
|
|
}
|
|
|
|
/**
|
|
* Float32Array를 PCM Blob으로 변환
|
|
*/
|
|
export function createPcmBlob(data) {
|
|
const l = data.length;
|
|
const int16 = new Int16Array(l);
|
|
for (let i = 0; i < l; i++) {
|
|
const s = Math.max(-1, Math.min(1, data[i]));
|
|
int16[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
|
|
}
|
|
return {
|
|
data: encodeBase64(new Uint8Array(int16.buffer)),
|
|
mimeType: 'audio/pcm;rate=16000',
|
|
};
|
|
}
|
|
|
|
// ============================================================================
|
|
// Connection Status
|
|
// ============================================================================
|
|
|
|
export const ConnectionStatus = {
|
|
DISCONNECTED: 'disconnected',
|
|
CONNECTING: 'connecting',
|
|
CONNECTED: 'connected',
|
|
ERROR: 'error',
|
|
};
|
|
|
|
// ============================================================================
|
|
// System Instruction
|
|
// ============================================================================
|
|
|
|
export const SYSTEM_INSTRUCTION = `You are "Sam", an intelligent and helpful corporate AI assistant.
|
|
Your goal is to help the user navigate the company dashboard and find work files efficiently.
|
|
You have access to tools that can control the UI.
|
|
|
|
1. If the user asks to see a specific page or menu (e.g., "Go to settings", "Show me analytics"), use the 'navigateToPage' tool.
|
|
2. If the user asks to find a file (e.g., "Find the financial report", "Search for marketing drafts"), use the 'searchDocuments' tool with their query.
|
|
3. Be concise, professional, but friendly.
|
|
4. You can speak Korean if the user speaks Korean. (한국어로 대답해 주세요).
|
|
5. If you perform an action, briefly confirm it verbally (e.g., "Opening Settings for you," or "Here are the files related to finance.").`;
|
|
|
|
// ============================================================================
|
|
// LiveManager Class
|
|
// ============================================================================
|
|
|
|
export class LiveManager {
|
|
constructor(apiKey, toolHandler, onStatusChange, onAudioLevel) {
|
|
this.ai = null;
|
|
this.inputAudioContext = null;
|
|
this.outputAudioContext = null;
|
|
this.sessionPromise = null;
|
|
this.session = null;
|
|
this.nextStartTime = 0;
|
|
this.sources = new Set();
|
|
this.toolHandler = toolHandler;
|
|
this.onStatusChange = onStatusChange;
|
|
this.onAudioLevel = onAudioLevel;
|
|
this.outputAnalyser = null;
|
|
this.apiKey = apiKey;
|
|
this.isConnected = false;
|
|
this.scriptProcessor = null;
|
|
this.mediaStream = null;
|
|
this.debugLog = [];
|
|
|
|
this.log = (msg, data = null) => {
|
|
const timestamp = new Date().toISOString();
|
|
const logEntry = { timestamp, msg, data };
|
|
this.debugLog.push(logEntry);
|
|
console.log(`[Sam AI Debug ${timestamp}]`, msg, data || '');
|
|
};
|
|
}
|
|
|
|
async initAI() {
|
|
this.log('initAI 시작', { hasApiKey: !!this.apiKey, apiKeyLength: this.apiKey?.length });
|
|
|
|
// GenAI SDK가 로드될 때까지 대기
|
|
if (!window.GoogleGenAI) {
|
|
this.log('GoogleGenAI SDK가 아직 로드되지 않음', { GenAILoaded: window.GenAILoaded });
|
|
|
|
if (window.GenAILoaded === false) {
|
|
this.log('Google GenAI SDK 로드 실패');
|
|
throw new Error('Google GenAI SDK 로드 실패');
|
|
}
|
|
|
|
this.log('Google GenAI SDK 로드 대기 중...');
|
|
await new Promise((resolve, reject) => {
|
|
const timeout = setTimeout(() => {
|
|
this.log('Google GenAI SDK 로드 타임아웃');
|
|
reject(new Error('Google GenAI SDK 로드 타임아웃'));
|
|
}, 10000);
|
|
|
|
if (window.GoogleGenAI) {
|
|
this.log('Google GenAI SDK 이미 로드됨');
|
|
clearTimeout(timeout);
|
|
resolve();
|
|
return;
|
|
}
|
|
|
|
const checkInterval = setInterval(() => {
|
|
if (window.GoogleGenAI) {
|
|
this.log('Google GenAI SDK 로드 완료 (interval)');
|
|
clearInterval(checkInterval);
|
|
clearTimeout(timeout);
|
|
resolve();
|
|
}
|
|
}, 100);
|
|
|
|
window.addEventListener('genai-loaded', () => {
|
|
this.log('Google GenAI SDK 로드 완료 (event)');
|
|
clearInterval(checkInterval);
|
|
clearTimeout(timeout);
|
|
resolve();
|
|
}, { once: true });
|
|
});
|
|
}
|
|
|
|
this.log('GoogleGenAI 인스턴스 생성 시작', { hasGoogleGenAI: !!window.GoogleGenAI });
|
|
this.ai = new window.GoogleGenAI({ apiKey: this.apiKey });
|
|
this.log('GoogleGenAI 인스턴스 생성 완료', { hasAI: !!this.ai });
|
|
}
|
|
|
|
async connect() {
|
|
this.log('=== connect() 시작 ===');
|
|
try {
|
|
await this.initAI();
|
|
this.log('initAI 완료, 연결 시작');
|
|
this.onStatusChange(ConnectionStatus.CONNECTING);
|
|
} catch (error) {
|
|
this.log('initAI 실패', { error: error.message, stack: error.stack });
|
|
throw error;
|
|
}
|
|
|
|
this.log('AudioContext 생성 시작');
|
|
try {
|
|
this.inputAudioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
|
|
this.log('inputAudioContext 생성 완료', { state: this.inputAudioContext.state, sampleRate: this.inputAudioContext.sampleRate });
|
|
} catch (error) {
|
|
this.log('inputAudioContext 생성 실패', { error: error.message });
|
|
throw error;
|
|
}
|
|
|
|
try {
|
|
this.outputAudioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 24000 });
|
|
this.log('outputAudioContext 생성 완료', { state: this.outputAudioContext.state, sampleRate: this.outputAudioContext.sampleRate });
|
|
} catch (error) {
|
|
this.log('outputAudioContext 생성 실패', { error: error.message });
|
|
throw error;
|
|
}
|
|
|
|
this.outputAnalyser = this.outputAudioContext.createAnalyser();
|
|
this.outputAnalyser.fftSize = 256;
|
|
this.outputAnalyser.connect(this.outputAudioContext.destination);
|
|
this.log('outputAnalyser 설정 완료');
|
|
|
|
const navigateToPage = {
|
|
name: 'navigateToPage',
|
|
parameters: {
|
|
type: window.Type.OBJECT,
|
|
description: 'Navigate the user to a specific page in the application menu.',
|
|
properties: {
|
|
pageId: {
|
|
type: window.Type.STRING,
|
|
description: 'The ID of the page to navigate to (e.g., dashboard, files, analytics, settings, team, calendar).',
|
|
},
|
|
},
|
|
required: ['pageId'],
|
|
},
|
|
};
|
|
|
|
const searchDocuments = {
|
|
name: 'searchDocuments',
|
|
parameters: {
|
|
type: window.Type.OBJECT,
|
|
description: 'Search for work documents or files based on a query.',
|
|
properties: {
|
|
query: {
|
|
type: window.Type.STRING,
|
|
description: 'The search keywords to filter files by.',
|
|
},
|
|
},
|
|
required: ['query'],
|
|
},
|
|
};
|
|
|
|
try {
|
|
this.log('live.connect() 호출 시작', {
|
|
hasAI: !!this.ai,
|
|
hasLive: !!this.ai?.live,
|
|
model: 'gemini-2.5-flash-native-audio-preview-09-2025'
|
|
});
|
|
|
|
this.sessionPromise = this.ai.live.connect({
|
|
model: 'gemini-2.5-flash-native-audio-preview-09-2025',
|
|
config: {
|
|
responseModalities: [window.Modality.AUDIO],
|
|
systemInstruction: SYSTEM_INSTRUCTION,
|
|
tools: [{ functionDeclarations: [navigateToPage, searchDocuments] }],
|
|
},
|
|
callbacks: {
|
|
onopen: () => {
|
|
this.log('WebSocket onopen 콜백 호출');
|
|
this.handleOpen();
|
|
},
|
|
onmessage: (message) => {
|
|
this.log('WebSocket onmessage 콜백 호출', {
|
|
hasServerContent: !!message.serverContent,
|
|
hasToolCall: !!message.toolCall
|
|
});
|
|
this.handleMessage(message);
|
|
},
|
|
onclose: (event) => {
|
|
this.log('WebSocket onclose 콜백 호출', {
|
|
code: event?.code,
|
|
reason: event?.reason,
|
|
wasClean: event?.wasClean
|
|
});
|
|
this.isConnected = false;
|
|
this.session = null;
|
|
this.cleanup();
|
|
this.onStatusChange(ConnectionStatus.DISCONNECTED);
|
|
},
|
|
onerror: (err) => {
|
|
this.log('WebSocket onerror 콜백 호출', {
|
|
error: err?.message || err,
|
|
errorType: err?.constructor?.name,
|
|
errorStack: err?.stack
|
|
});
|
|
console.error('WebSocket error:', err);
|
|
this.isConnected = false;
|
|
this.session = null;
|
|
this.cleanup();
|
|
this.onStatusChange(ConnectionStatus.ERROR);
|
|
},
|
|
},
|
|
});
|
|
|
|
this.log('live.connect() 호출 완료, Promise 반환됨', { hasPromise: !!this.sessionPromise });
|
|
|
|
this.sessionPromise.then(session => {
|
|
this.log('Session Promise resolved', { hasSession: !!session });
|
|
this.session = session;
|
|
}).catch(err => {
|
|
this.log('Session Promise rejected', {
|
|
error: err?.message || err,
|
|
errorType: err?.constructor?.name,
|
|
errorStack: err?.stack
|
|
});
|
|
console.error("Session promise failed", err);
|
|
this.isConnected = false;
|
|
this.onStatusChange(ConnectionStatus.ERROR);
|
|
});
|
|
} catch (error) {
|
|
this.log('live.connect() 호출 중 예외 발생', {
|
|
error: error.message,
|
|
errorType: error.constructor?.name,
|
|
errorStack: error.stack
|
|
});
|
|
console.error("Connection failed", error);
|
|
this.isConnected = false;
|
|
this.onStatusChange(ConnectionStatus.ERROR);
|
|
}
|
|
}
|
|
|
|
async handleOpen() {
|
|
this.log('=== handleOpen() 시작 ===');
|
|
this.isConnected = true;
|
|
this.log('isConnected = true 설정');
|
|
this.onStatusChange(ConnectionStatus.CONNECTED);
|
|
this.log('상태를 connected로 변경');
|
|
|
|
try {
|
|
this.log('getUserMedia 호출 시작');
|
|
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
this.mediaStream = stream;
|
|
this.log('getUserMedia 성공', {
|
|
hasStream: !!stream,
|
|
activeTracks: stream.getTracks().filter(t => t.readyState === 'live').length
|
|
});
|
|
|
|
if (!this.inputAudioContext) {
|
|
this.log('inputAudioContext가 없음, cleanup 호출');
|
|
this.cleanup();
|
|
return;
|
|
}
|
|
|
|
this.log('MediaStreamSource 생성 시작');
|
|
const source = this.inputAudioContext.createMediaStreamSource(stream);
|
|
this.log('ScriptProcessor 생성 시작');
|
|
this.scriptProcessor = this.inputAudioContext.createScriptProcessor(4096, 1, 1);
|
|
|
|
this.scriptProcessor.onaudioprocess = (e) => {
|
|
if (!this.isConnected || !this.session) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const inputData = e.inputBuffer.getChannelData(0);
|
|
let sum = 0;
|
|
for (let i = 0; i < inputData.length; i++) sum += inputData[i] * inputData[i];
|
|
this.onAudioLevel(Math.sqrt(sum / inputData.length));
|
|
|
|
const pcmBlob = createPcmBlob(inputData);
|
|
|
|
if (this.session && this.isConnected) {
|
|
try {
|
|
this.session.sendRealtimeInput({ media: pcmBlob });
|
|
} catch (sendError) {
|
|
if (sendError.message && (sendError.message.includes('CLOSING') || sendError.message.includes('CLOSED'))) {
|
|
this.log('WebSocket이 닫힘, 연결 종료', { error: sendError.message });
|
|
this.isConnected = false;
|
|
this.cleanup();
|
|
return;
|
|
}
|
|
console.warn('Send error:', sendError);
|
|
}
|
|
}
|
|
} catch (err) {
|
|
console.warn('Audio process error:', err);
|
|
}
|
|
};
|
|
|
|
this.log('오디오 소스 연결 시작');
|
|
source.connect(this.scriptProcessor);
|
|
this.scriptProcessor.connect(this.inputAudioContext.destination);
|
|
this.log('오디오 소스 연결 완료');
|
|
} catch (err) {
|
|
this.log('getUserMedia 실패', {
|
|
error: err.message,
|
|
errorName: err.name,
|
|
errorStack: err.stack
|
|
});
|
|
console.error("Microphone access denied", err);
|
|
this.isConnected = false;
|
|
this.cleanup();
|
|
this.onStatusChange(ConnectionStatus.ERROR);
|
|
}
|
|
}
|
|
|
|
async handleMessage(message) {
|
|
const base64Audio = message.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data;
|
|
if (base64Audio && this.outputAudioContext && this.outputAnalyser) {
|
|
if (this.outputAudioContext.state === 'suspended') {
|
|
await this.outputAudioContext.resume();
|
|
}
|
|
|
|
const audioBuffer = await decodeAudioData(
|
|
decodeBase64(base64Audio),
|
|
this.outputAudioContext,
|
|
24000
|
|
);
|
|
|
|
this.nextStartTime = Math.max(this.outputAudioContext.currentTime, this.nextStartTime);
|
|
|
|
const source = this.outputAudioContext.createBufferSource();
|
|
source.buffer = audioBuffer;
|
|
source.connect(this.outputAnalyser);
|
|
source.start(this.nextStartTime);
|
|
|
|
this.nextStartTime += audioBuffer.duration;
|
|
this.sources.add(source);
|
|
|
|
source.onended = () => this.sources.delete(source);
|
|
}
|
|
|
|
if (message.toolCall) {
|
|
for (const fc of message.toolCall.functionCalls) {
|
|
console.log(`Tool Call: ${fc.name}`, fc.args);
|
|
let result;
|
|
try {
|
|
result = await this.toolHandler(fc.name, fc.args);
|
|
} catch (e) {
|
|
result = { error: 'Failed to execute tool' };
|
|
}
|
|
|
|
if (this.session && this.isConnected) {
|
|
try {
|
|
this.session.sendToolResponse({
|
|
functionResponses: {
|
|
id: fc.id,
|
|
name: fc.name,
|
|
response: { result },
|
|
}
|
|
});
|
|
} catch (sendError) {
|
|
console.warn('Tool response send error:', sendError);
|
|
if (sendError.message && (sendError.message.includes('CLOSING') || sendError.message.includes('CLOSED'))) {
|
|
this.isConnected = false;
|
|
this.cleanup();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (message.serverContent?.interrupted) {
|
|
this.sources.forEach(s => s.stop());
|
|
this.sources.clear();
|
|
this.nextStartTime = 0;
|
|
}
|
|
}
|
|
|
|
cleanup() {
|
|
this.log('=== cleanup() 시작 ===');
|
|
|
|
if (this.mediaStream) {
|
|
this.log('미디어 스트림 트랙 정지');
|
|
this.mediaStream.getTracks().forEach(track => {
|
|
this.log('트랙 정지', { kind: track.kind, label: track.label, enabled: track.enabled });
|
|
track.stop();
|
|
});
|
|
this.mediaStream = null;
|
|
}
|
|
|
|
if (this.scriptProcessor) {
|
|
this.log('ScriptProcessor 연결 해제');
|
|
try {
|
|
this.scriptProcessor.disconnect();
|
|
} catch (e) {
|
|
this.log('ScriptProcessor disconnect 오류 (무시)', { error: e.message });
|
|
}
|
|
this.scriptProcessor = null;
|
|
}
|
|
|
|
this.log('오디오 소스 정리', { sourceCount: this.sources.size });
|
|
this.sources.forEach(s => {
|
|
try {
|
|
s.stop();
|
|
} catch (e) {
|
|
// 이미 정지된 경우 무시
|
|
}
|
|
});
|
|
this.sources.clear();
|
|
|
|
if (this.inputAudioContext) {
|
|
this.log('inputAudioContext 종료', { state: this.inputAudioContext.state });
|
|
try {
|
|
this.inputAudioContext.close();
|
|
} catch (e) {
|
|
this.log('inputAudioContext close 오류 (무시)', { error: e.message });
|
|
}
|
|
this.inputAudioContext = null;
|
|
}
|
|
|
|
if (this.outputAudioContext) {
|
|
this.log('outputAudioContext 종료', { state: this.outputAudioContext.state });
|
|
try {
|
|
this.outputAudioContext.close();
|
|
} catch (e) {
|
|
this.log('outputAudioContext close 오류 (무시)', { error: e.message });
|
|
}
|
|
this.outputAudioContext = null;
|
|
}
|
|
|
|
if (this.session && typeof this.session.close === 'function') {
|
|
this.log('세션 종료');
|
|
try {
|
|
this.session.close();
|
|
} catch (e) {
|
|
this.log('세션 close 오류 (무시)', { error: e.message });
|
|
}
|
|
}
|
|
this.session = null;
|
|
this.sessionPromise = null;
|
|
this.log('=== cleanup() 완료 ===');
|
|
}
|
|
|
|
disconnect() {
|
|
this.isConnected = false;
|
|
this.cleanup();
|
|
this.onStatusChange(ConnectionStatus.DISCONNECTED);
|
|
}
|
|
|
|
getOutputAnalyser() {
|
|
return this.outputAnalyser;
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// Audio Visualizer
|
|
// ============================================================================
|
|
|
|
export class AudioVisualizer {
|
|
constructor(canvas) {
|
|
this.canvas = canvas;
|
|
this.ctx = canvas.getContext('2d');
|
|
this.analyser = null;
|
|
this.active = false;
|
|
this.animationId = null;
|
|
}
|
|
|
|
setAnalyser(analyser) {
|
|
this.analyser = analyser;
|
|
}
|
|
|
|
setActive(active) {
|
|
this.active = active;
|
|
}
|
|
|
|
start() {
|
|
const bufferLength = this.analyser ? this.analyser.frequencyBinCount : 0;
|
|
const dataArray = new Uint8Array(bufferLength);
|
|
|
|
const draw = () => {
|
|
this.animationId = requestAnimationFrame(draw);
|
|
|
|
const width = this.canvas.width;
|
|
const height = this.canvas.height;
|
|
this.ctx.clearRect(0, 0, width, height);
|
|
|
|
if (!this.active || !this.analyser) {
|
|
const time = Date.now() / 1000;
|
|
this.ctx.beginPath();
|
|
this.ctx.arc(width / 2, height / 2, 20 + Math.sin(time * 2) * 2, 0, Math.PI * 2);
|
|
this.ctx.fillStyle = 'rgba(99, 102, 241, 0.5)';
|
|
this.ctx.fill();
|
|
return;
|
|
}
|
|
|
|
this.analyser.getByteFrequencyData(dataArray);
|
|
|
|
const barWidth = (width / bufferLength) * 2.5;
|
|
let barHeight;
|
|
let x = 0;
|
|
const centerX = width / 2;
|
|
|
|
for (let i = 0; i < bufferLength; i++) {
|
|
barHeight = dataArray[i] / 2;
|
|
|
|
const gradient = this.ctx.createLinearGradient(0, height / 2 - barHeight, 0, height / 2 + barHeight);
|
|
gradient.addColorStop(0, '#818cf8');
|
|
gradient.addColorStop(1, '#c084fc');
|
|
|
|
this.ctx.fillStyle = gradient;
|
|
this.ctx.fillRect(centerX + x, height / 2 - barHeight / 2, barWidth, barHeight);
|
|
this.ctx.fillRect(centerX - x - barWidth, height / 2 - barHeight / 2, barWidth, barHeight);
|
|
|
|
x += barWidth + 1;
|
|
}
|
|
};
|
|
|
|
draw();
|
|
}
|
|
|
|
stop() {
|
|
if (this.animationId) {
|
|
cancelAnimationFrame(this.animationId);
|
|
this.animationId = null;
|
|
}
|
|
}
|
|
}
|