#!/usr/bin/env node /** * E2E 전체 테스트 독립 실행 러너 * * MCP 오버헤드 없이 Node.js + Playwright 직접 사용 * 시나리오당 ~10초, 전체 ~15분 목표 * * Usage: * node e2e/runner/run-all.js # 전체 실행 * node e2e/runner/run-all.js --filter board # 파일명 필터 * node e2e/runner/run-all.js --workflow # 워크플로우만 실행 * node e2e/runner/run-all.js --headed # headed (기본값) * node e2e/runner/run-all.js --headless # headless * node e2e/runner/run-all.js --exclude sales # 파일명에 "sales" 포함된 것 제외 */ const fs = require('fs'); const path = require('path'); // ─── Resolve Playwright from react/node_modules ───────────── const SAM_ROOT = path.resolve(__dirname, '..', '..'); const PW_PATH = path.join(SAM_ROOT, 'react', 'node_modules', 'playwright'); const { chromium } = require(PW_PATH); // ─── Config ───────────────────────────────────────────────── const BASE_URL = 'https://dev.codebridge-x.com'; const AUTH = { username: 'TestUser5', password: 'password123!' }; const SCENARIOS_DIR = path.join(SAM_ROOT, 'e2e', 'scenarios'); const RESULTS_DIR = path.join(SAM_ROOT, 'e2e', 'results', 'hotfix'); const SUCCESS_DIR = path.join(RESULTS_DIR, 'success'); const SCREENSHOTS_DIR = path.join(RESULTS_DIR, 'screenshots'); const EXECUTOR_PATH = path.join(SAM_ROOT, 'e2e', 'runner', 'step-executor.js'); const DASHBOARD_URL = `${BASE_URL}/dashboard`; // CLI args const args = process.argv.slice(2); const HEADLESS = args.includes('--headless'); const WORKFLOW_ONLY = args.includes('--workflow'); const FILTER = (() => { const idx = args.indexOf('--filter'); return idx >= 0 && args[idx + 1] ? args[idx + 1] : null; })(); const EXCLUDE = (() => { const idx = args.indexOf('--exclude'); return idx >= 0 && args[idx + 1] ? args[idx + 1] : null; })(); // ─── Helpers ──────────────────────────────────────────────── function getTimestamp() { const n = new Date(); const pad = (v) => v.toString().padStart(2, '0'); return `${n.getFullYear()}-${pad(n.getMonth() + 1)}-${pad(n.getDate())}_${pad(n.getHours())}-${pad(n.getMinutes())}-${pad(n.getSeconds())}`; } function ensureDir(dir) { if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); } function sleep(ms) { return new Promise((r) => setTimeout(r, ms)); } /** Color console helpers */ const C = { green: (s) => `\x1b[32m${s}\x1b[0m`, red: (s) => `\x1b[31m${s}\x1b[0m`, yellow: (s) => `\x1b[33m${s}\x1b[0m`, cyan: (s) => `\x1b[36m${s}\x1b[0m`, dim: (s) => `\x1b[2m${s}\x1b[0m`, bold: (s) => `\x1b[1m${s}\x1b[0m`, }; // ─── Executor Injection ───────────────────────────────────── const executorCode = fs.readFileSync(EXECUTOR_PATH, 'utf-8'); async function injectExecutor(page) { await page.evaluate(executorCode); const initResult = await page.evaluate(() => JSON.stringify(window.__E2E__.init())); const parsed = JSON.parse(initResult); if (!parsed.ready) throw new Error('step-executor init failed'); return parsed; } // ─── Menu Navigation ──────────────────────────────────────── /** * Wait for sidebar to render with clickable menu items. * Returns true if sidebar is ready, false otherwise. */ async function waitForSidebarReady(page, timeout = 8000) { try { await page.waitForFunction( () => { const sidebar = document.querySelector('.sidebar-scroll, [data-sidebar="content"]'); if (!sidebar) return false; const items = sidebar.querySelectorAll('a, button, [role="button"], [role="menuitem"]'); return Array.from(items).filter(el => { const t = (el.innerText || '').trim(); return t.length > 1 && t.length < 30; }).length >= 3; }, null, { timeout } ); return true; } catch (e) { return false; } } /** * Ensure sidebar is expanded (not icon-only mode). * Checks localStorage and reloads from Node.js side if needed. */ async function ensureSidebarExpanded(page) { const needsReload = await page.evaluate(() => { try { const raw = localStorage.getItem('sam-menu'); if (raw) { const data = JSON.parse(raw); if (data.state && data.state.sidebarCollapsed) { data.state.sidebarCollapsed = false; localStorage.setItem('sam-menu', JSON.stringify(data)); return true; // needs reload to apply } } } catch (e) {} return false; }); if (needsReload) { await page.reload({ waitUntil: 'load', timeout: 12000 }); await sleep(1500); } } async function navigateViaMenu(page, level1, level2) { // Phase 0: Ensure sidebar is rendered and expanded let sidebarReady = await waitForSidebarReady(page, 6000); if (!sidebarReady) { console.log(C.yellow(` [NAV] sidebar not ready, reloading...`)); try { await page.reload({ waitUntil: 'load', timeout: 12000 }); await sleep(2000); } catch (e) { /* ignore */ } sidebarReady = await waitForSidebarReady(page, 8000); if (!sidebarReady) { console.log(C.red(` [NAV] sidebar still not rendered after reload! URL: ${page.url()}`)); return false; } } await ensureSidebarExpanded(page); // Phase 1: Collapse all open accordions, scroll to top await page.evaluate(() => { const collapseBtn = Array.from(document.querySelectorAll('button, [role="button"]')) .find(el => el.innerText?.trim() === '모두 접기'); if (collapseBtn) collapseBtn.click(); }); await sleep(400); await page.evaluate(() => { const sidebar = document.querySelector('.sidebar-scroll, [data-sidebar="content"], nav'); if (sidebar) sidebar.scrollTo({ top: 0, behavior: 'instant' }); }); await sleep(300); // Phase 2: Find and click L1 menu (accordion header) // Use innerText for accurate visible-text matching (textContent includes hidden child text) const l1Found = await page.evaluate( async ({ l1Text }) => { const sidebar = document.querySelector('.sidebar-scroll, [data-sidebar="content"], nav'); const maxScrollAttempts = 25; for (let i = 0; i < maxScrollAttempts; i++) { // Collect only direct menu buttons/links (not their children) const candidates = Array.from( document.querySelectorAll( '[data-sidebar="content"] > * > * > button, ' + '[data-sidebar="content"] > * > * > a, ' + '.sidebar-scroll button, .sidebar-scroll a, ' + 'nav button, nav a, ' + '[role="menuitem"], [role="treeitem"]' ) ); for (const el of candidates) { // Use innerText for accurate match (excludes hidden sub-menus) const elText = (el.innerText || '').trim(); // Also check only the first line (in case submenu text is appended) const firstLine = elText.split('\n')[0].trim(); if (firstLine === l1Text || elText === l1Text) { el.scrollIntoView({ behavior: 'instant', block: 'center' }); await new Promise(r => setTimeout(r, 150)); el.click(); return { found: true, text: firstLine, attempt: i }; } } // Fallback: startsWith match (e.g., "회계관리 14" badge suffix) for (const el of candidates) { const firstLine = (el.innerText || '').split('\n')[0].trim(); if (firstLine.startsWith(l1Text) && firstLine.length < l1Text.length + 10) { el.scrollIntoView({ behavior: 'instant', block: 'center' }); await new Promise(r => setTimeout(r, 150)); el.click(); return { found: true, text: firstLine, attempt: i, partial: true }; } } if (sidebar) { sidebar.scrollBy({ top: 120, behavior: 'instant' }); await new Promise(r => setTimeout(r, 120)); } } return { found: false }; }, { l1Text: level1 } ); if (!l1Found.found) { // Collect debug info before returning const debugTexts = await page.evaluate(() => { const items = document.querySelectorAll('nav a, nav button, [role="menuitem"], [role="treeitem"]'); return Array.from(items) .map(el => (el.innerText || '').split('\n')[0].trim()) .filter(t => t.length > 1 && t.length < 25) .filter((t, i, arr) => arr.indexOf(t) === i) .slice(0, 20); }); console.log(C.red(` [NAV] L1 "${level1}" not found.`)); console.log(C.dim(` [NAV] Available: ${debugTexts.join(', ')}`)); return false; } await sleep(800); // Wait for accordion animation to expand // Phase 3: Verify accordion expanded (L2 items should now be visible) if (level2) { // Wait for L2 items to appear after accordion expansion let l2Visible = false; for (let retryWait = 0; retryWait < 3; retryWait++) { l2Visible = await page.evaluate( (l2Text) => { const items = document.querySelectorAll('a, button, [role="menuitem"], [role="treeitem"]'); return Array.from(items).some(el => { const t = (el.innerText || '').trim(); return t === l2Text || t.includes(l2Text); }); }, level2 ); if (l2Visible) break; // Accordion might not have expanded - try clicking L1 again if (retryWait === 1) { await page.evaluate( async ({ l1Text }) => { const candidates = document.querySelectorAll('nav button, nav a, [role="menuitem"], [role="treeitem"]'); for (const el of candidates) { const firstLine = (el.innerText || '').split('\n')[0].trim(); if (firstLine === l1Text || firstLine.startsWith(l1Text)) { el.click(); break; } } }, { l1Text: level1 } ); } await sleep(600); } // Phase 4: Find and click L2 menu item const l2Found = await page.evaluate( async ({ l2Text }) => { const sidebar = document.querySelector('.sidebar-scroll, [data-sidebar="content"], nav'); const maxAttempts = 15; for (let i = 0; i < maxAttempts; i++) { const items = Array.from( document.querySelectorAll('a, button, [role="menuitem"], [role="treeitem"]') ); // Exact match on innerText (most reliable) let match = items.find(el => (el.innerText || '').trim() === l2Text); // Partial match fallback if (!match) { match = items.find(el => { const t = (el.innerText || '').trim(); return t.includes(l2Text) && t.length < l2Text.length + 15; }); } if (match) { match.scrollIntoView({ behavior: 'instant', block: 'center' }); await new Promise(r => setTimeout(r, 150)); match.click(); return { found: true }; } if (sidebar) { sidebar.scrollBy({ top: 100, behavior: 'instant' }); await new Promise(r => setTimeout(r, 120)); } } return { found: false }; }, { l2Text: level2 } ); if (!l2Found.found) { console.log(C.red(` [NAV] L2 "${level2}" not found under "${level1}".`)); return false; } await sleep(2000); // Wait for page load after L2 click } return true; } /** * navigateViaMenu with automatic retry (up to 2 attempts). */ async function navigateViaMenuWithRetry(page, level1, level2, maxRetries = 2) { for (let attempt = 1; attempt <= maxRetries; attempt++) { const ok = await navigateViaMenu(page, level1, level2); if (ok) return true; if (attempt < maxRetries) { console.log(C.yellow(` [NAV] Retry ${attempt}/${maxRetries - 1} for ${level1} > ${level2}`)); // Go back to dashboard and try again try { await page.goto(DASHBOARD_URL, { waitUntil: 'load', timeout: 12000 }); await sleep(1500); await ensureSidebarExpanded(page); await waitForSidebarReady(page, 6000); } catch (e) { await sleep(2000); } } } return false; } // ─── Dashboard Navigation ─────────────────────────────────── async function ensureLoggedIn(page) { const url = page.url(); if (url.includes('/login')) { // Need to re-login await page.fill('#userId', AUTH.username); await page.fill('#password', AUTH.password); await page.click("button[type='submit']"); await sleep(3000); } } async function goToDashboard(page) { // Force sidebar expanded state BEFORE navigation so page loads with full menu try { await page.evaluate(() => { try { const raw = localStorage.getItem('sam-menu'); if (raw) { const data = JSON.parse(raw); if (data.state) { data.state.sidebarCollapsed = false; localStorage.setItem('sam-menu', JSON.stringify(data)); } } } catch (e) {} }); } catch (e) { /* page may not be ready yet */ } // Attempt 1: Navigate to dashboard try { await page.goto(DASHBOARD_URL, { waitUntil: 'load', timeout: 15000 }); await sleep(1000); await ensureLoggedIn(page); await ensureSidebarExpanded(page); await waitForSidebarReady(page, 8000); return; } catch (e) { const dbgUrl = page.url(); console.log(C.yellow(` [DASH] attempt 1 failed. URL: ${dbgUrl}, err: ${e.message?.substring(0, 80)}`)); } // Attempt 2: Reload to force fresh render try { await page.reload({ waitUntil: 'load', timeout: 10000 }); await sleep(1500); await ensureLoggedIn(page); await ensureSidebarExpanded(page); await waitForSidebarReady(page, 8000); return; } catch (e) { const dbgUrl = page.url(); console.log(C.yellow(` [DASH] reload failed. URL: ${dbgUrl}, err: ${e.message?.substring(0, 80)}`)); } // Attempt 3: Full re-login try { await page.goto(`${BASE_URL}/ko/login`, { waitUntil: 'load', timeout: 10000 }); await sleep(500); try { await page.fill('#userId', AUTH.username); await page.fill('#password', AUTH.password); await page.click("button[type='submit']"); await sleep(3000); } catch (loginErr) { /* may already be on dashboard */ } await ensureSidebarExpanded(page); await waitForSidebarReady(page, 8000); } catch (e) { await sleep(2000); } } const SCENARIO_TIMEOUT = 180000; // 3 minutes per scenario (batch-create needs extra time) const WORKFLOW_TIMEOUT = 300000; // 5 minutes for workflow scenarios (multi-module chains) const PERFORMANCE_TIMEOUT = 120000; // 2 minutes for performance scenarios /** Determine timeout based on scenario category */ function getScenarioTimeout(filename) { if (filename.startsWith('workflow-')) return WORKFLOW_TIMEOUT; if (filename.startsWith('perf-')) return PERFORMANCE_TIMEOUT; return SCENARIO_TIMEOUT; } /** Classify scenario into a category for summary grouping */ function getScenarioCategory(filename) { if (filename.startsWith('workflow-')) return 'workflow'; if (filename.startsWith('perf-')) return 'performance'; if (filename.startsWith('edge-')) return 'edge-case'; if (filename.startsWith('a11y-')) return 'accessibility'; return 'functional'; } // ─── Page Health Verify ───────────────────────────────────── /** * Pre-flight page health check. * Runs AFTER menu navigation, BEFORE step execution. * Detects page crashes, console errors, Error Boundaries, API failures. * * Returns: { healthy: boolean, diagnosis: { ... } } */ async function verifyPageHealth(page, timeout = 8000) { const diagnosis = { healthy: true, url: '', crashed: false, errorBoundary: null, consoleErrors: [], apiErrors: [], emptySelectValues: 0, blankPage: false, loadTimeout: false, }; try { diagnosis.url = page.url(); // 1. Collect console errors captured during page load // (we attach listener before navigation in runScenario, store them on page object) diagnosis.consoleErrors = (page.__e2e_console_errors || []).slice(-10); // 2. Check for Error Boundary / crash screen / blank page const pageState = await Promise.race([ page.evaluate(() => { const bodyText = document.body?.innerText || ''; // Error Boundary patterns (React) // NOTE: bodyText(innerText)만 사용. innerHTML은 i18n 번역 JSON에 // "서버 오류가 발생했습니다" 등이 포함되어 false positive 발생함 const errorBoundaryPatterns = [ '오류가 발생했습니다', '일시적인 오류', 'Something went wrong', 'Error boundary', 'An error occurred', 'Unhandled Runtime Error', 'Application error', ]; const foundErrorText = errorBoundaryPatterns.find(p => bodyText.includes(p) ); // Blank page detection const hasContent = bodyText.trim().length > 50; const hasMainContent = !!document.querySelector( 'table, [class*="content"], main, [role="main"], [class*="page"], [class*="list"]' ); // Console errors (if captured by step-executor ApiMonitor) const apiErrors = (window.__API_ERRORS__ || []).filter(e => e.status >= 400 || e.error ); // Empty Select.Item values (the exact bug pattern) const emptySelectItems = document.querySelectorAll( '[role="option"][data-value=""], select option[value=""]' ); // Check for React error overlay (dev mode) const reactErrorOverlay = document.querySelector( '[data-nextjs-dialog], #__next-build-error, [class*="nextjs-container-errors"]' ); return { foundErrorText, hasContent, hasMainContent, apiErrorCount: apiErrors.length, apiErrors: apiErrors.slice(0, 5).map(e => ({ url: (e.url || '').substring(0, 100), status: e.status, method: e.method, error: e.error, })), emptySelectItems: emptySelectItems.length, reactErrorOverlay: !!reactErrorOverlay, reactErrorMsg: reactErrorOverlay?.innerText?.substring(0, 200) || null, }; }), sleep(timeout).then(() => ({ timeout: true })), ]); if (pageState.timeout) { diagnosis.healthy = false; diagnosis.loadTimeout = true; return diagnosis; } // Error Boundary detected if (pageState.foundErrorText) { diagnosis.healthy = false; diagnosis.crashed = true; diagnosis.errorBoundary = pageState.foundErrorText; } // React error overlay (dev/next.js) if (pageState.reactErrorOverlay) { diagnosis.healthy = false; diagnosis.crashed = true; diagnosis.errorBoundary = pageState.reactErrorMsg || 'React Error Overlay detected'; } // Blank page if (!pageState.hasContent && !pageState.hasMainContent) { diagnosis.healthy = false; diagnosis.blankPage = true; } // API errors if (pageState.apiErrorCount > 0) { diagnosis.apiErrors = pageState.apiErrors; // API 500 errors make the page unhealthy if (pageState.apiErrors.some(e => e.status >= 500)) { diagnosis.healthy = false; } } // Empty Select values (Radix UI crash pattern) diagnosis.emptySelectValues = pageState.emptySelectItems; if (pageState.emptySelectItems > 0) { diagnosis.healthy = false; } // Console errors make it unhealthy if they contain crash indicators const criticalConsolePatterns = [ 'Select.Item', 'must have a value', 'Uncaught', 'ChunkLoadError', 'Cannot read properties of null', 'Cannot read properties of undefined', 'Maximum update depth exceeded', 'Minified React error', ]; const criticalConsoleErrors = diagnosis.consoleErrors.filter(msg => criticalConsolePatterns.some(p => msg.includes(p)) ); if (criticalConsoleErrors.length > 0) { diagnosis.healthy = false; } } catch (err) { diagnosis.healthy = false; diagnosis.crashed = true; diagnosis.errorBoundary = `Health check error: ${err.message}`; } return diagnosis; } /** * Post-failure diagnosis. * Runs AFTER a scenario fails to collect detailed root cause information. * Captures: console errors, DOM state, API logs, screenshot. * * Returns: { rootCause: string, details: { ... } } */ async function diagnoseFail(page, result, scenarioId) { const diag = { rootCause: 'unknown', consoleErrors: [], apiErrors: [], pageState: null, screenshotPath: null, recommendations: [], }; try { // 1. Capture screenshot const ssName = `diag_${scenarioId}_${getTimestamp()}.png`; const ssPath = path.join(SCREENSHOTS_DIR, ssName); try { await page.screenshot({ path: ssPath, fullPage: false, timeout: 5000 }); diag.screenshotPath = ssPath; } catch (e) { /* screenshot failed, continue */ } // 2. Collect console errors diag.consoleErrors = (page.__e2e_console_errors || []).slice(-20); // 3. Collect page state & API errors try { const state = await Promise.race([ page.evaluate(() => { const bodyText = document.body?.innerText || ''; // Error Boundary check const errorPatterns = [ '오류가 발생했습니다', '일시적인 오류', 'Something went wrong', 'Error boundary', ]; const errorBoundary = errorPatterns.find(p => bodyText.includes(p)); // API errors from step-executor monitor const apiLogs = window.__API_LOGS__ || []; const apiErrors = (window.__API_ERRORS__ || []).slice(0, 10).map(e => ({ url: (e.url || '').substring(0, 120), status: e.status, method: e.method, error: e.error, })); // Null data detection in rendered content const nullPatterns = bodyText.match(/null|undefined/gi) || []; // DOM stats const domNodes = document.getElementsByTagName('*').length; const tables = document.querySelectorAll('table'); const tableRowCount = tables.length > 0 ? tables[0].querySelectorAll('tbody tr').length : 0; const hasLoadingSpinner = !!document.querySelector( '.loading, .spinner, [class*="skeleton"], [class*="loading"], [class*="Skeleton"]' ); return { url: window.location.href, errorBoundary, apiTotal: apiLogs.length, apiErrors, nullCount: nullPatterns.length, domNodes, tableRowCount, hasLoadingSpinner, visibleText: bodyText.substring(0, 300), }; }), sleep(5000).then(() => null), ]); if (state) { diag.pageState = state; diag.apiErrors = state.apiErrors; // Classify root cause if (state.errorBoundary) { diag.rootCause = 'page_crash'; diag.recommendations.push('페이지 크래시 - Error Boundary 활성화됨. Console 에러 확인 필요'); } else if (state.apiErrors.some(e => e.status >= 500)) { diag.rootCause = 'api_server_error'; diag.recommendations.push('백엔드 서버 에러 (5xx). 서버 로그 확인 필요'); } else if (state.apiErrors.some(e => e.status === 401 || e.status === 403)) { diag.rootCause = 'auth_error'; diag.recommendations.push('인증/권한 에러. 세션 만료 가능성'); } else if (state.hasLoadingSpinner) { diag.rootCause = 'infinite_loading'; diag.recommendations.push('무한 로딩 상태. API 미응답 또는 프론트엔드 상태 관리 버그'); } else if (state.tableRowCount === 0 && state.apiTotal > 0) { diag.rootCause = 'empty_data'; diag.recommendations.push('API 응답은 있으나 테이블 데이터 없음. 데이터 변환 또는 필터 문제'); } } } catch (e) { diag.rootCause = 'page_unresponsive'; diag.recommendations.push('페이지 응답 없음 (evaluate 실패). 네비게이션 에러 또는 크래시'); } // 4. Console error pattern matching for specific root causes const consoleText = diag.consoleErrors.join(' '); if (consoleText.includes('Select.Item') || consoleText.includes('must have a value')) { diag.rootCause = 'select_empty_value'; diag.recommendations = ['Radix Select.Item에 빈 value 전달됨. 데이터 transform에서 null/빈값 방어 필요']; } else if (consoleText.includes('ChunkLoadError') || consoleText.includes('Loading chunk')) { diag.rootCause = 'chunk_load_error'; diag.recommendations = ['JS 번들 로드 실패. 배포 상태 또는 네트워크 문제']; } else if (consoleText.includes('Cannot read properties of null') || consoleText.includes('Cannot read properties of undefined')) { diag.rootCause = 'null_reference'; diag.recommendations = ['Null 참조 에러. API 응답에서 예상치 못한 null 데이터 가능성']; } else if (consoleText.includes('Maximum update depth')) { diag.rootCause = 'infinite_render_loop'; diag.recommendations = ['React 무한 렌더 루프. useEffect 의존성 배열 또는 상태 업데이트 로직 확인']; } // 5. Check failed step patterns for additional context if (result.steps) { const failedSteps = result.steps.filter(s => s.status === 'fail'); const timeoutSteps = failedSteps.filter(s => (s.error || s.details || '').includes('timeout') || (s.error || s.details || '').includes('Timeout') ); if (timeoutSteps.length > 0 && diag.rootCause === 'unknown') { diag.rootCause = 'element_timeout'; diag.recommendations.push('요소 대기 타임아웃. 페이지 로드 지연 또는 셀렉터 불일치'); } } } catch (err) { diag.rootCause = 'diagnosis_error'; diag.recommendations.push(`진단 중 에러: ${err.message}`); } return diag; } // ─── Scenario Runner ──────────────────────────────────────── async function runScenario(page, scenarioPath) { const scenarioJson = JSON.parse(fs.readFileSync(scenarioPath, 'utf-8')); const { id, name, steps, selectors, menuNavigation } = scenarioJson; const result = { id: id || path.basename(scenarioPath, '.json'), name: name || id, steps: [], passed: 0, failed: 0, warned: 0, totalSteps: 0, apiSummary: null, error: null, stoppedReason: 'complete', currentUrl: '', startTime: Date.now(), endTime: 0, healthCheck: null, // Page health verification result diagnosis: null, // Post-failure diagnosis result }; // Attach console error listener for this scenario page.__e2e_console_errors = []; const consoleHandler = (msg) => { if (msg.type() === 'error') { page.__e2e_console_errors.push(msg.text().substring(0, 500)); } }; page.on('console', consoleHandler); // Also capture uncaught page errors const pageErrorHandler = (err) => { page.__e2e_console_errors.push(`[PAGE_ERROR] ${err.message}`.substring(0, 500)); }; page.on('pageerror', pageErrorHandler); if (!steps || steps.length === 0) { result.error = 'No steps defined'; result.stoppedReason = 'no_steps'; result.endTime = Date.now(); page.removeListener('console', consoleHandler); page.removeListener('pageerror', pageErrorHandler); return result; } try { // Navigate to dashboard first await goToDashboard(page); // Inject executor await injectExecutor(page); // Menu navigation if specified if (menuNavigation && menuNavigation.level1) { const navOk = await navigateViaMenuWithRetry(page, menuNavigation.level1, menuNavigation.level2); if (!navOk) { result.error = `Menu navigation failed: ${menuNavigation.level1} > ${menuNavigation.level2}`; result.stoppedReason = 'navigation_failed'; result.endTime = Date.now(); page.removeListener('console', consoleHandler); page.removeListener('pageerror', pageErrorHandler); return result; } // Re-inject after navigation await sleep(1000); await injectExecutor(page); } // ─── Page Health Check (Pre-flight) ─────────────────── const health = await verifyPageHealth(page); result.healthCheck = health; if (!health.healthy) { // Page is unhealthy - run diagnosis immediately and abort const diag = await diagnoseFail(page, result, result.id); result.diagnosis = diag; // Build descriptive error message const reasons = []; if (health.crashed || health.errorBoundary) reasons.push(`페이지 크래시: ${health.errorBoundary}`); if (health.blankPage) reasons.push('빈 페이지 (콘텐츠 없음)'); if (health.emptySelectValues > 0) reasons.push(`빈 Select 값 ${health.emptySelectValues}개 감지`); if (health.loadTimeout) reasons.push('페이지 로드 타임아웃'); if (health.apiErrors.length > 0) reasons.push(`API 에러 ${health.apiErrors.length}건 (${health.apiErrors.map(e => `${e.status} ${e.method}`).join(', ')})`); const consoleSnippet = (health.consoleErrors || []) .filter(msg => msg.length > 10) .slice(0, 3) .map(msg => msg.substring(0, 150)); result.error = `[HEALTH_CHECK] ${reasons.join(' | ')}`; if (consoleSnippet.length > 0) { result.error += ` | Console: ${consoleSnippet.join('; ')}`; } result.stoppedReason = 'health_check_failed'; console.log(C.yellow(` [HEALTH] ✘ ${reasons[0] || 'unhealthy'}`)); if (diag.rootCause !== 'unknown') { console.log(C.yellow(` [DIAG] root cause: ${diag.rootCause}`)); } result.endTime = Date.now(); page.removeListener('console', consoleHandler); page.removeListener('pageerror', pageErrorHandler); return result; } // Run steps in batches (handling navigation stops) let currentIndex = 0; let vars = {}; const allResults = []; while (currentIndex < steps.length) { const batch = steps.slice(currentIndex); let batchResult; try { batchResult = await page.evaluate( async ({ batch, vars, selectors }) => { const r = await window.__E2E__.runBatch(batch, vars, { selectors: selectors || {} }); return r; }, { batch, vars, selectors: selectors || {} } ); } catch (evalErr) { // If evaluate fails (page navigated away, etc.), try re-injection try { await sleep(2000); await injectExecutor(page); batchResult = await page.evaluate( async ({ batch, vars, selectors }) => { const r = await window.__E2E__.runBatch(batch, vars, { selectors: selectors || {} }); return r; }, { batch, vars, selectors: selectors || {} } ); } catch (retryErr) { result.error = `Evaluate failed: ${retryErr.message}`; result.stoppedReason = 'evaluate_error'; break; } } // Collect results if (batchResult.results) { allResults.push(...batchResult.results); } vars = batchResult.variables || vars; result.apiSummary = batchResult.apiSummary; result.currentUrl = batchResult.currentUrl; // Handle stoppedReason if (batchResult.stoppedReason === 'complete') { break; } if (batchResult.stoppedReason === 'navigation') { await sleep(2000); try { await injectExecutor(page); } catch (e) { // Page might still be loading await sleep(3000); try { await injectExecutor(page); } catch (e2) { result.error = `Re-injection failed after navigation: ${e2.message}`; result.stoppedReason = 'reinjection_failed'; break; } } currentIndex += batchResult.stoppedAtIndex; continue; } if (batchResult.stoppedReason === 'native_required') { // Take screenshot const ssName = `${result.id}_step${batchResult.stoppedAtIndex}_${getTimestamp()}.png`; try { await page.screenshot({ path: path.join(SCREENSHOTS_DIR, ssName), fullPage: false }); } catch (ssErr) { // Screenshot failed, continue } currentIndex += batchResult.stoppedAtIndex + 1; // Re-inject in case page changed try { await injectExecutor(page); } catch (e) { /* ignore */ } continue; } if (batchResult.stoppedReason === 'critical_failure') { result.stoppedReason = 'critical_failure'; break; } // Unknown stoppedReason break; } // Aggregate results result.steps = allResults; result.totalSteps = allResults.length; result.passed = allResults.filter((r) => r.status === 'pass').length; result.failed = allResults.filter((r) => r.status === 'fail').length; result.warned = allResults.filter((r) => r.status === 'warn').length; if (result.stoppedReason === 'complete' && result.failed > 0) { result.stoppedReason = 'completed_with_failures'; } // ─── Post-failure Diagnosis ──────────────────────────── if (result.failed > 0 || result.error) { try { const diag = await diagnoseFail(page, result, result.id); result.diagnosis = diag; if (diag.rootCause !== 'unknown') { console.log(C.yellow(` [DIAG] root cause: ${diag.rootCause}`)); } } catch (diagErr) { // Diagnosis itself failed, don't block } } } catch (err) { result.error = err.message; result.stoppedReason = 'exception'; // Try diagnosis even on exception try { const diag = await diagnoseFail(page, result, result.id); result.diagnosis = diag; } catch (diagErr) { /* ignore */ } } // Cleanup event listeners page.removeListener('console', consoleHandler); page.removeListener('pageerror', pageErrorHandler); result.endTime = Date.now(); return result; } // ─── Report Generation ────────────────────────────────────── function generateReport(result, timestamp) { const duration = ((result.endTime - result.startTime) / 1000).toFixed(1); const hasFail = result.failed > 0 || result.error; const status = hasFail ? 'FAIL' : 'PASS'; const icon = hasFail ? '❌' : '✅'; const stepsTable = result.steps .map((s) => { const statusIcon = s.status === 'pass' ? '✅' : s.status === 'fail' ? '❌' : '⚠️'; const phase = s.phase || '-'; const details = (s.details || '').substring(0, 80).replace(/\|/g, '/'); return `| ${s.stepId} | ${s.name} | ${phase} | ${statusIcon} | ${s.duration}ms | ${details} |`; }) .join('\n'); const failedSteps = result.steps.filter((s) => s.status === 'fail'); const failedTable = failedSteps.length > 0 ? failedSteps .map((s) => `| ${s.stepId} | ${s.name} | ${s.phase || '-'} | ${(s.error || s.details || '').substring(0, 100)} |`) .join('\n') : ''; const api = result.apiSummary || { total: 0, success: 0, failed: 0, avgResponseTime: 0, slowCalls: 0 }; let md = `# ${icon} E2E 테스트 ${hasFail ? '실패' : '성공'}: ${result.name} **테스트 ID**: ${result.id} | **실행**: ${timestamp} | **결과**: ${status} **소요 시간**: ${duration}초${result.error ? ` | **에러**: ${result.error}` : ''}${result.stoppedReason !== 'complete' && result.stoppedReason !== 'completed_with_failures' ? ` | **중단 사유**: ${result.stoppedReason}` : ''} ## 테스트 요약 | 전체 | 성공 | 실패 | 경고 | 성공률 | |------|------|------|------|--------| | ${result.totalSteps} | ${result.passed} | ${result.failed} | ${result.warned} | ${result.totalSteps > 0 ? Math.round((result.passed / result.totalSteps) * 100) : 0}% | `; if (failedTable) { md += ` ## 실패 스텝 | # | 스텝 | Phase | 에러 | |---|------|-------|------| ${failedTable} `; } md += ` ## 전체 스텝 결과 | # | 스텝 | Phase | 상태 | 소요시간 | 비고 | |---|------|-------|------|---------|------| ${stepsTable || '| - | (스텝 없음) | - | - | - | - |'} ## API 요약 | 총 호출 | 성공 | 실패 | 평균 응답 | 느린 호출(>2s) | |---------|------|------|----------|--------------| | ${api.total} | ${api.success} | ${api.failed} | ${api.avgResponseTime}ms | ${api.slowCalls} | `; // Health Check section if (result.healthCheck) { const h = result.healthCheck; md += '\n## 페이지 건강 검사\n'; md += '| 항목 | 결과 |\n|------|------|\n'; md += '| 상태 | ' + (h.healthy ? '✅ 정상' : '❌ 비정상') + ' |\n'; md += '| URL | ' + (h.url || '-') + ' |\n'; if (h.crashed) md += '| 크래시 | ' + (h.errorBoundary || 'Yes') + ' |\n'; if (h.blankPage) md += '| 빈 페이지 | Yes |\n'; if (h.loadTimeout) md += '| 로드 타임아웃 | Yes |\n'; if (h.emptySelectValues > 0) md += '| 빈 Select 값 | ' + h.emptySelectValues + '개 |\n'; if (h.apiErrors && h.apiErrors.length > 0) { const apiErrStr = h.apiErrors.map(function(e) { return e.status + ' ' + e.method + ' ' + e.url; }).join(', '); md += '| API 에러 | ' + apiErrStr + ' |\n'; } if (h.consoleErrors && h.consoleErrors.length > 0) { md += '\n### 콘솔 에러 (Health Check)\n'; h.consoleErrors.slice(0, 5).forEach(function(err, i) { md += (i + 1) + '. `' + err.substring(0, 200) + '`\n'; }); } } // Diagnosis section if (result.diagnosis) { const d = result.diagnosis; md += '\n## 자동 진단\n'; md += '| 항목 | 내용 |\n|------|------|\n'; md += '| 근본 원인 | **' + d.rootCause + '** |\n'; if (d.screenshotPath) md += '| 스크린샷 | ' + path.basename(d.screenshotPath) + ' |\n'; if (d.recommendations && d.recommendations.length > 0) { md += '\n### 권장 조치\n'; d.recommendations.forEach(function(rec, i) { md += (i + 1) + '. ' + rec + '\n'; }); } if (d.consoleErrors && d.consoleErrors.length > 0) { md += '\n### 콘솔 에러 (진단)\n'; d.consoleErrors.slice(0, 10).forEach(function(err, i) { md += (i + 1) + '. `' + err.substring(0, 200) + '`\n'; }); } if (d.pageState) { const ps = d.pageState; md += '\n### 페이지 상태\n'; md += '| 항목 | 값 |\n|------|----|\n'; md += '| DOM 노드 | ' + (ps.domNodes || '-') + ' |\n'; md += '| 테이블 행 | ' + (ps.tableRowCount || 0) + ' |\n'; md += '| API 호출 수 | ' + (ps.apiTotal || 0) + ' |\n'; md += '| 로딩 스피너 | ' + (ps.hasLoadingSpinner ? 'Yes' : 'No') + ' |\n'; if (ps.errorBoundary) md += '| Error Boundary | ' + ps.errorBoundary + ' |\n'; } } return md; } function saveReport(result, timestamp) { const hasFail = result.failed > 0 || result.error; if (hasFail) { const filePath = path.join(RESULTS_DIR, `Fail-${result.id}_${timestamp}.md`); fs.writeFileSync(filePath, generateReport(result, timestamp), 'utf-8'); return filePath; } else { const filePath = path.join(SUCCESS_DIR, `OK-${result.id}_${timestamp}.md`); fs.writeFileSync(filePath, generateReport(result, timestamp), 'utf-8'); return filePath; } } // ─── Summary Report ───────────────────────────────────────── function generateSummaryReport(allResults, totalTime, timestamp) { const passed = allResults.filter((r) => !r.error && r.failed === 0).length; const failed = allResults.length - passed; // Categorize results const categories = {}; allResults.forEach((r) => { const cat = getScenarioCategory(r.id || ''); if (!categories[cat]) categories[cat] = []; categories[cat].push(r); }); let md = `# E2E 전체 테스트 결과 요약 **실행 시간**: ${timestamp} **총 소요 시간**: ${(totalTime / 1000 / 60).toFixed(1)}분 **전체 시나리오**: ${allResults.length}개 | **성공**: ${passed}개 | **실패**: ${failed}개 ## 카테고리별 요약 | 카테고리 | 시나리오 수 | 성공 | 실패 | 성공률 | |---------|-----------|------|------|--------| `; const catNames = { functional: '기능 테스트', workflow: '비즈니스 워크플로우', performance: '성능 테스트', 'edge-case': '엣지 케이스', accessibility: '접근성 검사' }; for (const [cat, results] of Object.entries(categories)) { const catPassed = results.filter(r => !r.error && r.failed === 0).length; const catFailed = results.length - catPassed; const rate = results.length > 0 ? Math.round((catPassed / results.length) * 100) : 0; md += `| ${catNames[cat] || cat} | ${results.length} | ${catPassed} | ${catFailed} | ${rate}% |\n`; } md += ` ## 시나리오별 결과 | # | 시나리오 | 결과 | 스텝 | 성공 | 실패 | 소요(초) | |---|---------|------|------|------|------|---------| `; allResults.forEach((r, i) => { const hasFail = r.failed > 0 || r.error; const icon = hasFail ? '❌' : '✅'; const duration = ((r.endTime - r.startTime) / 1000).toFixed(1); md += `| ${i + 1} | ${r.name} | ${icon} | ${r.totalSteps} | ${r.passed} | ${r.failed} | ${duration} |\n`; }); // Workflow summary section if (categories.workflow && categories.workflow.length > 0) { md += `\n## 비즈니스 워크플로우 상세\n`; categories.workflow.forEach((r) => { const hasFail = r.failed > 0 || r.error; const icon = hasFail ? '❌' : '✅'; const duration = ((r.endTime - r.startTime) / 1000).toFixed(1); md += `\n### ${icon} ${r.name}\n`; md += `- 스텝: ${r.passed}/${r.totalSteps} 성공 | 소요: ${duration}초\n`; if (r.error) md += `- 에러: ${r.error}\n`; const phases = r.steps.filter(s => s.phase).map(s => `${s.phase}(${s.status === 'pass' ? '✅' : '❌'})`); if (phases.length > 0) md += `- 단계: ${phases.join(' → ')}\n`; }); } // Performance summary section if (categories.performance && categories.performance.length > 0) { md += `\n## 성능 테스트 요약\n`; md += `| 페이지 | 로드 시간 | 등급 | API 평균 | DOM 노드 |\n`; md += `|--------|----------|------|---------|----------|\n`; categories.performance.forEach((r) => { const perfStep = r.steps.find(s => s.phase === 'PERF_MEASURE'); const perfData = perfStep?.details ? (() => { try { return JSON.parse(perfStep.details); } catch(e) { return null; } })() : null; if (perfData) { md += `| ${r.name} | ${perfData.loadTime || '-'}ms | ${perfData.grade || '-'} | ${perfData.apiAvg || '-'}ms | ${perfData.domNodes || '-'} |\n`; } else { md += `| ${r.name} | - | - | - | - |\n`; } }); } // Accessibility summary section if (categories.accessibility && categories.accessibility.length > 0) { md += `\n## 접근성 검사 요약\n`; md += `| 페이지 | 점수 | 등급 | Critical | Serious | Moderate |\n`; md += `|--------|------|------|----------|---------|----------|\n`; categories.accessibility.forEach((r) => { const a11yStep = r.steps.find(s => s.phase === 'A11Y_AUDIT'); const a11yData = a11yStep?.details ? (() => { try { return JSON.parse(a11yStep.details); } catch(e) { return null; } })() : null; if (a11yData) { md += `| ${r.name} | ${a11yData.score || '-'} | ${a11yData.grade || '-'} | ${a11yData.critical || 0} | ${a11yData.serious || 0} | ${a11yData.moderate || 0} |\n`; } else { md += `| ${r.name} | - | - | - | - | - |\n`; } }); } if (failed > 0) { md += `\n## 실패 시나리오 상세\n`; allResults .filter((r) => r.failed > 0 || r.error) .forEach((r) => { md += `\n### ❌ ${r.name} (${r.id})\n`; if (r.error) md += `- **에러**: ${r.error}\n`; // Diagnosis info if (r.diagnosis && r.diagnosis.rootCause !== 'unknown') { md += `- **진단**: ${r.diagnosis.rootCause}`; if (r.diagnosis.recommendations && r.diagnosis.recommendations.length > 0) { md += ` → ${r.diagnosis.recommendations[0]}`; } md += `\n`; } // Health check info if (r.healthCheck && !r.healthCheck.healthy) { const h = r.healthCheck; const issues = []; if (h.crashed) issues.push('크래시'); if (h.blankPage) issues.push('빈 페이지'); if (h.loadTimeout) issues.push('로드 타임아웃'); if (h.emptySelectValues > 0) issues.push(`빈 Select 값 ${h.emptySelectValues}개`); if (issues.length > 0) md += `- **건강 검사**: ${issues.join(', ')}\n`; } const failSteps = r.steps.filter((s) => s.status === 'fail'); failSteps.forEach((s) => { md += `- Step ${s.stepId} (${s.name}): ${s.error || s.details}\n`; }); }); } return md; } // ─── Main ─────────────────────────────────────────────────── async function main() { console.log(C.bold('\n=== E2E 전체 테스트 러너 ===')); console.log(`서버: ${BASE_URL}`); console.log(`모드: ${HEADLESS ? 'headless' : 'headed'}`); if (WORKFLOW_ONLY) console.log(`카테고리: workflow only`); if (FILTER) console.log(`필터: ${FILTER}`); if (EXCLUDE) console.log(`제외: ${EXCLUDE}`); console.log(''); // Ensure directories ensureDir(RESULTS_DIR); ensureDir(SUCCESS_DIR); ensureDir(SCREENSHOTS_DIR); // Collect scenario files (skip disabled scenarios) let scenarioFiles = fs.readdirSync(SCENARIOS_DIR) .filter((f) => f.endsWith('.json') && !f.startsWith('_')) .filter((f) => { try { const data = JSON.parse(fs.readFileSync(path.join(SCENARIOS_DIR, f), 'utf-8')); return data.enabled !== false; // Skip if explicitly disabled } catch (e) { return true; // Include if can't parse (will fail later with better error) } }) .sort(); if (WORKFLOW_ONLY) { scenarioFiles = scenarioFiles.filter((f) => f.startsWith('workflow-')); } if (FILTER) { scenarioFiles = scenarioFiles.filter((f) => f.includes(FILTER)); } if (EXCLUDE) { scenarioFiles = scenarioFiles.filter((f) => !f.includes(EXCLUDE)); } const totalScenarios = scenarioFiles.length; console.log(`시나리오: ${totalScenarios}개 발견\n`); if (totalScenarios === 0) { console.log(C.red('실행할 시나리오가 없습니다.')); process.exit(1); } // Launch browser const browser = await chromium.launch({ headless: HEADLESS, args: [ `--window-position=1920,0`, '--window-size=1920,1080', '--disable-blink-features=AutomationControlled', ], }); const context = await browser.newContext({ viewport: { width: 1920, height: 1080 }, locale: 'ko-KR', ignoreHTTPSErrors: true, userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', }); const page = await context.newPage(); // Mask automation detection await page.addInitScript(() => { Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); }); // Login console.log(C.cyan('로그인 중...')); try { await page.goto(`${BASE_URL}/ko/login`, { waitUntil: 'domcontentloaded', timeout: 20000 }); await sleep(1000); await page.fill('#userId', AUTH.username); await page.fill('#password', AUTH.password); await page.click("button[type='submit']"); await sleep(3000); const url = page.url(); if (url.includes('/login')) { console.log(C.red('로그인 실패! 대시보드로 이동하지 못함')); await browser.close(); process.exit(1); } console.log(C.green('로그인 성공!\n')); } catch (loginErr) { console.log(C.red(`로그인 에러: ${loginErr.message}`)); await browser.close(); process.exit(1); } // Run scenarios const allResults = []; const startTime = Date.now(); for (let i = 0; i < scenarioFiles.length; i++) { const file = scenarioFiles[i]; const scenarioPath = path.join(SCENARIOS_DIR, file); const num = `(${i + 1}/${totalScenarios})`; process.stdout.write(`${C.dim(num)} ${file.replace('.json', '')} ... `); let result; const timeout = getScenarioTimeout(file); try { // Wrap with timeout result = await Promise.race([ runScenario(page, scenarioPath), sleep(timeout).then(() => ({ id: file.replace('.json', ''), name: file.replace('.json', ''), steps: [], passed: 0, failed: 0, warned: 0, totalSteps: 0, apiSummary: null, error: `Timeout (>${timeout / 1000}s)`, stoppedReason: 'timeout', currentUrl: '', startTime: Date.now() - timeout, endTime: Date.now(), })), ]); } catch (scenarioErr) { result = { id: file.replace('.json', ''), name: file.replace('.json', ''), steps: [], passed: 0, failed: 0, warned: 0, totalSteps: 0, apiSummary: null, error: scenarioErr.message, stoppedReason: 'exception', currentUrl: '', startTime: Date.now(), endTime: Date.now(), }; } allResults.push(result); // Save report const ts = getTimestamp(); saveReport(result, ts); // Console output const hasFail = result.failed > 0 || result.error; const duration = ((result.endTime - result.startTime) / 1000).toFixed(1); if (hasFail) { console.log(`${C.red('FAIL')} ${C.dim(`(${result.passed}/${result.totalSteps} passed, ${duration}s)`)}`); } else { console.log(`${C.green('PASS')} ${C.dim(`(${result.passed}/${result.totalSteps}, ${duration}s)`)}`); } // After login scenario or if logged out, re-login if (file === 'login.json' || page.url().includes('/login')) { try { await page.goto(`${BASE_URL}/ko/login`, { waitUntil: 'domcontentloaded', timeout: 10000 }); await sleep(500); await page.fill('#userId', AUTH.username); await page.fill('#password', AUTH.password); await page.click("button[type='submit']"); await sleep(3000); } catch (reloginErr) { // Continue anyway } } } const totalTime = Date.now() - startTime; // Generate summary report const summaryTs = getTimestamp(); const summaryMd = generateSummaryReport(allResults, totalTime, summaryTs); const summaryPath = path.join(RESULTS_DIR, `E2E_FULL_TEST_SUMMARY_${summaryTs}.md`); fs.writeFileSync(summaryPath, summaryMd, 'utf-8'); // Close browser await browser.close(); // Print summary const passCount = allResults.filter((r) => !r.error && r.failed === 0).length; const failCount = allResults.length - passCount; console.log(C.bold('\n=== 테스트 완료 ===')); console.log(`전체: ${totalScenarios} | ${C.green(`성공: ${passCount}`)} | ${failCount > 0 ? C.red(`실패: ${failCount}`) : '실패: 0'}`); console.log(`소요 시간: ${(totalTime / 1000 / 60).toFixed(1)}분`); console.log(`요약 리포트: ${summaryPath}`); console.log(''); } main().catch((err) => { console.error(C.red(`\n치명적 에러: ${err.message}`)); console.error(err.stack); process.exit(1); });