3e9c638b04
- --fail-only: 최근 요약 리포트에서 실패 시나리오만 추출하여 집중 실행 - getFailedScenarioIds(): E2E_FULL_TEST_SUMMARY에서 실패 ID 파싱 - --fail-only --iterate 조합으로 실패분만 반복 타격 가능 - 8차 전체 검증: 184개 중 176 PASS / 8 FAIL (95.7%) - 9차 실패 집중 검증: 8개 × 4라운드 모두 FAIL (확정적 앱 버그) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1754 lines
65 KiB
JavaScript
1754 lines
65 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* E2E 전체 테스트 독립 실행 러너
|
|
*
|
|
* MCP 오버헤드 없이 Node.js + Playwright 직접 사용
|
|
* 시나리오당 ~10초, 전체 ~15분 목표
|
|
*
|
|
* Usage:
|
|
* node e2e/runner/run-all.js # 전체 실행
|
|
* node e2e/runner/run-all.js --filter board # 파일명 필터
|
|
* node e2e/runner/run-all.js --workflow # 워크플로우만 실행
|
|
* node e2e/runner/run-all.js --headed # headed (기본값)
|
|
* node e2e/runner/run-all.js --headless # headless
|
|
* node e2e/runner/run-all.js --exclude sales # 파일명에 "sales" 포함된 것 제외
|
|
* node e2e/runner/run-all.js --skip-passed # 이미 성공한 시나리오 건너뛰기
|
|
* node e2e/runner/run-all.js --iterate # 실패 시나리오 자동 재실행 (최대 3회)
|
|
* node e2e/runner/run-all.js --iterate 5 # 실패 시나리오 자동 재실행 (최대 5회)
|
|
* node e2e/runner/run-all.js --stage # 카테고리별 단계 실행 (accessibility → edge → perf → workflow → functional)
|
|
* node e2e/runner/run-all.js --fail-only # 최근 요약 리포트의 실패 시나리오만 실행
|
|
* node e2e/runner/run-all.js --fail-only --iterate # 실패 시나리오만 반복 실행
|
|
*/
|
|
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
// ─── Resolve Playwright from react/node_modules ─────────────
|
|
const SAM_ROOT = path.resolve(__dirname, '..', '..');
|
|
const PW_PATH = path.join(SAM_ROOT, 'react', 'node_modules', 'playwright');
|
|
const { chromium } = require(PW_PATH);
|
|
|
|
// ─── Config ─────────────────────────────────────────────────
|
|
const BASE_URL = 'https://dev.codebridge-x.com';
|
|
const AUTH = { username: 'TestUser5', password: 'password123!' };
|
|
const SCENARIOS_DIR = path.join(SAM_ROOT, 'e2e', 'scenarios');
|
|
const RESULTS_DIR = path.join(SAM_ROOT, 'e2e', 'results', 'hotfix');
|
|
const SUCCESS_DIR = path.join(RESULTS_DIR, 'success');
|
|
const SCREENSHOTS_DIR = path.join(RESULTS_DIR, 'screenshots');
|
|
const EXECUTOR_PATH = path.join(SAM_ROOT, 'e2e', 'runner', 'step-executor.js');
|
|
const DASHBOARD_URL = `${BASE_URL}/dashboard`;
|
|
|
|
// CLI args
|
|
const args = process.argv.slice(2);
|
|
const HEADLESS = args.includes('--headless');
|
|
const WORKFLOW_ONLY = args.includes('--workflow');
|
|
const FILTER = (() => {
|
|
const idx = args.indexOf('--filter');
|
|
return idx >= 0 && args[idx + 1] ? args[idx + 1] : null;
|
|
})();
|
|
const EXCLUDE = (() => {
|
|
const idx = args.indexOf('--exclude');
|
|
return idx >= 0 && args[idx + 1] ? args[idx + 1] : null;
|
|
})();
|
|
const SKIP_PASSED = args.includes('--skip-passed');
|
|
const ITERATE = args.includes('--iterate');
|
|
const MAX_ITERATIONS = (() => {
|
|
const idx = args.indexOf('--iterate');
|
|
if (idx >= 0 && args[idx + 1] && !args[idx + 1].startsWith('--')) {
|
|
const n = parseInt(args[idx + 1], 10);
|
|
return isNaN(n) ? 3 : Math.min(Math.max(n, 1), 10);
|
|
}
|
|
return 3;
|
|
})();
|
|
const STAGE_MODE = args.includes('--stage');
|
|
const FAIL_ONLY = args.includes('--fail-only');
|
|
|
|
// ─── Helpers ────────────────────────────────────────────────
|
|
|
|
function getTimestamp() {
|
|
const n = new Date();
|
|
const pad = (v) => v.toString().padStart(2, '0');
|
|
return `${n.getFullYear()}-${pad(n.getMonth() + 1)}-${pad(n.getDate())}_${pad(n.getHours())}-${pad(n.getMinutes())}-${pad(n.getSeconds())}`;
|
|
}
|
|
|
|
function ensureDir(dir) {
|
|
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
}
|
|
|
|
function sleep(ms) {
|
|
return new Promise((r) => setTimeout(r, ms));
|
|
}
|
|
|
|
/**
|
|
* success/ 폴더의 OK- 리포트에서 이미 성공한 시나리오 ID 집합을 반환.
|
|
* 파일명 형식: OK-{scenario-id}_{timestamp}.md
|
|
*/
|
|
function getPassedScenarioIds() {
|
|
const passed = new Set();
|
|
if (!fs.existsSync(SUCCESS_DIR)) return passed;
|
|
const files = fs.readdirSync(SUCCESS_DIR).filter(f => f.startsWith('OK-') && f.endsWith('.md'));
|
|
for (const f of files) {
|
|
// OK-accounting-deposit_2026-02-23_20-22-27.md → accounting-deposit
|
|
const withoutPrefix = f.substring(3); // remove "OK-"
|
|
const tsMatch = withoutPrefix.match(/_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\.md$/);
|
|
if (tsMatch) {
|
|
const id = withoutPrefix.substring(0, tsMatch.index);
|
|
passed.add(id);
|
|
}
|
|
}
|
|
return passed;
|
|
}
|
|
|
|
/**
|
|
* 최근 E2E_FULL_TEST_SUMMARY 리포트에서 실패한 시나리오 ID 집합을 반환.
|
|
* --fail-only 모드에서 사용.
|
|
*
|
|
* 파싱 전략:
|
|
* 1차: "### ❌ {name} ({scenario-id})" 형식에서 ID 추출 (가장 신뢰)
|
|
* 2차: Fail-{scenario-id}_{timestamp}.md 파일명에서 추출 (보완)
|
|
*/
|
|
function getFailedScenarioIds() {
|
|
const failed = new Set();
|
|
if (!fs.existsSync(RESULTS_DIR)) return { ids: failed, source: null };
|
|
|
|
// E2E_FULL_TEST_SUMMARY_*.md 파일 중 가장 최근 것 찾기
|
|
const summaryFiles = fs.readdirSync(RESULTS_DIR)
|
|
.filter(f => f.startsWith('E2E_FULL_TEST_SUMMARY_') && f.endsWith('.md'))
|
|
.sort()
|
|
.reverse();
|
|
|
|
if (summaryFiles.length === 0) {
|
|
console.log(' (이전 요약 리포트 없음 - Fail- 파일에서 추출)');
|
|
// Fail- 리포트에서 추출
|
|
const failFiles = fs.readdirSync(RESULTS_DIR)
|
|
.filter(f => f.startsWith('Fail-') && f.endsWith('.md'));
|
|
for (const f of failFiles) {
|
|
const withoutPrefix = f.substring(5); // remove "Fail-"
|
|
const tsMatch = withoutPrefix.match(/_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\.md$/);
|
|
if (tsMatch) {
|
|
failed.add(withoutPrefix.substring(0, tsMatch.index));
|
|
}
|
|
}
|
|
return { ids: failed, source: 'Fail-*.md files' };
|
|
}
|
|
|
|
const latestSummary = summaryFiles[0];
|
|
const content = fs.readFileSync(path.join(RESULTS_DIR, latestSummary), 'utf-8');
|
|
|
|
// "### ❌ 시나리오명 (scenario-id)" 패턴에서 ID 추출
|
|
const lines = content.split('\n');
|
|
for (const line of lines) {
|
|
const match = line.match(/^###\s*❌\s*.+\(([a-zA-Z0-9_-]+)\)\s*$/);
|
|
if (match) {
|
|
failed.add(match[1]);
|
|
}
|
|
}
|
|
|
|
// 보완: Fail- 리포트에서도 추출 (요약 파싱 실패 시)
|
|
if (failed.size === 0) {
|
|
const failFiles = fs.readdirSync(RESULTS_DIR)
|
|
.filter(f => f.startsWith('Fail-') && f.endsWith('.md'));
|
|
for (const f of failFiles) {
|
|
const withoutPrefix = f.substring(5);
|
|
const tsMatch = withoutPrefix.match(/_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\.md$/);
|
|
if (tsMatch) {
|
|
failed.add(withoutPrefix.substring(0, tsMatch.index));
|
|
}
|
|
}
|
|
}
|
|
|
|
return { ids: failed, source: latestSummary };
|
|
}
|
|
|
|
/** Color console helpers */
|
|
const C = {
|
|
green: (s) => `\x1b[32m${s}\x1b[0m`,
|
|
red: (s) => `\x1b[31m${s}\x1b[0m`,
|
|
yellow: (s) => `\x1b[33m${s}\x1b[0m`,
|
|
cyan: (s) => `\x1b[36m${s}\x1b[0m`,
|
|
dim: (s) => `\x1b[2m${s}\x1b[0m`,
|
|
bold: (s) => `\x1b[1m${s}\x1b[0m`,
|
|
};
|
|
|
|
// ─── Executor Injection ─────────────────────────────────────
|
|
|
|
const executorCode = fs.readFileSync(EXECUTOR_PATH, 'utf-8');
|
|
|
|
async function injectExecutor(page) {
|
|
await page.evaluate(executorCode);
|
|
const initResult = await page.evaluate(() => JSON.stringify(window.__E2E__.init()));
|
|
const parsed = JSON.parse(initResult);
|
|
if (!parsed.ready) throw new Error('step-executor init failed');
|
|
return parsed;
|
|
}
|
|
|
|
// ─── Menu Navigation ────────────────────────────────────────
|
|
|
|
/**
|
|
* Wait for sidebar to render with clickable menu items.
|
|
* Returns true if sidebar is ready, false otherwise.
|
|
*/
|
|
async function waitForSidebarReady(page, timeout = 8000) {
|
|
try {
|
|
await page.waitForFunction(
|
|
() => {
|
|
const sidebar = document.querySelector('.sidebar-scroll, [data-sidebar="content"]');
|
|
if (!sidebar) return false;
|
|
const items = sidebar.querySelectorAll('a, button, [role="button"], [role="menuitem"]');
|
|
return Array.from(items).filter(el => {
|
|
const t = (el.innerText || '').trim();
|
|
return t.length > 1 && t.length < 30;
|
|
}).length >= 3;
|
|
},
|
|
null,
|
|
{ timeout }
|
|
);
|
|
return true;
|
|
} catch (e) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Ensure sidebar is expanded (not icon-only mode).
|
|
* Checks localStorage and reloads from Node.js side if needed.
|
|
*/
|
|
async function ensureSidebarExpanded(page) {
|
|
const needsReload = await page.evaluate(() => {
|
|
try {
|
|
const raw = localStorage.getItem('sam-menu');
|
|
if (raw) {
|
|
const data = JSON.parse(raw);
|
|
if (data.state && data.state.sidebarCollapsed) {
|
|
data.state.sidebarCollapsed = false;
|
|
localStorage.setItem('sam-menu', JSON.stringify(data));
|
|
return true; // needs reload to apply
|
|
}
|
|
}
|
|
} catch (e) {}
|
|
return false;
|
|
});
|
|
|
|
if (needsReload) {
|
|
await page.reload({ waitUntil: 'load', timeout: 12000 });
|
|
await sleep(1500);
|
|
}
|
|
}
|
|
|
|
async function navigateViaMenu(page, level1, level2) {
|
|
// Phase 0: Ensure sidebar is rendered and expanded
|
|
let sidebarReady = await waitForSidebarReady(page, 6000);
|
|
if (!sidebarReady) {
|
|
console.log(C.yellow(` [NAV] sidebar not ready, reloading...`));
|
|
try {
|
|
await page.reload({ waitUntil: 'load', timeout: 12000 });
|
|
await sleep(2000);
|
|
} catch (e) { /* ignore */ }
|
|
sidebarReady = await waitForSidebarReady(page, 8000);
|
|
if (!sidebarReady) {
|
|
console.log(C.red(` [NAV] sidebar still not rendered after reload! URL: ${page.url()}`));
|
|
return false;
|
|
}
|
|
}
|
|
|
|
await ensureSidebarExpanded(page);
|
|
|
|
// Phase 1: Collapse all open accordions, scroll to top
|
|
await page.evaluate(() => {
|
|
const collapseBtn = Array.from(document.querySelectorAll('button, [role="button"]'))
|
|
.find(el => el.innerText?.trim() === '모두 접기');
|
|
if (collapseBtn) collapseBtn.click();
|
|
});
|
|
await sleep(400);
|
|
|
|
await page.evaluate(() => {
|
|
const sidebar = document.querySelector('.sidebar-scroll, [data-sidebar="content"], nav');
|
|
if (sidebar) sidebar.scrollTo({ top: 0, behavior: 'instant' });
|
|
});
|
|
await sleep(300);
|
|
|
|
// Phase 2: Find and click L1 menu (accordion header)
|
|
// Use innerText for accurate visible-text matching (textContent includes hidden child text)
|
|
const l1Found = await page.evaluate(
|
|
async ({ l1Text }) => {
|
|
const sidebar = document.querySelector('.sidebar-scroll, [data-sidebar="content"], nav');
|
|
const maxScrollAttempts = 25;
|
|
|
|
for (let i = 0; i < maxScrollAttempts; i++) {
|
|
// Collect only direct menu buttons/links (not their children)
|
|
const candidates = Array.from(
|
|
document.querySelectorAll(
|
|
'[data-sidebar="content"] > * > * > button, ' +
|
|
'[data-sidebar="content"] > * > * > a, ' +
|
|
'.sidebar-scroll button, .sidebar-scroll a, ' +
|
|
'nav button, nav a, ' +
|
|
'[role="menuitem"], [role="treeitem"]'
|
|
)
|
|
);
|
|
|
|
for (const el of candidates) {
|
|
// Use innerText for accurate match (excludes hidden sub-menus)
|
|
const elText = (el.innerText || '').trim();
|
|
// Also check only the first line (in case submenu text is appended)
|
|
const firstLine = elText.split('\n')[0].trim();
|
|
|
|
if (firstLine === l1Text || elText === l1Text) {
|
|
el.scrollIntoView({ behavior: 'instant', block: 'center' });
|
|
await new Promise(r => setTimeout(r, 150));
|
|
el.click();
|
|
return { found: true, text: firstLine, attempt: i };
|
|
}
|
|
}
|
|
|
|
// Fallback: startsWith match (e.g., "회계관리 14" badge suffix)
|
|
for (const el of candidates) {
|
|
const firstLine = (el.innerText || '').split('\n')[0].trim();
|
|
if (firstLine.startsWith(l1Text) && firstLine.length < l1Text.length + 10) {
|
|
el.scrollIntoView({ behavior: 'instant', block: 'center' });
|
|
await new Promise(r => setTimeout(r, 150));
|
|
el.click();
|
|
return { found: true, text: firstLine, attempt: i, partial: true };
|
|
}
|
|
}
|
|
|
|
if (sidebar) {
|
|
sidebar.scrollBy({ top: 120, behavior: 'instant' });
|
|
await new Promise(r => setTimeout(r, 120));
|
|
}
|
|
}
|
|
return { found: false };
|
|
},
|
|
{ l1Text: level1 }
|
|
);
|
|
|
|
if (!l1Found.found) {
|
|
// Collect debug info before returning
|
|
const debugTexts = await page.evaluate(() => {
|
|
const items = document.querySelectorAll('nav a, nav button, [role="menuitem"], [role="treeitem"]');
|
|
return Array.from(items)
|
|
.map(el => (el.innerText || '').split('\n')[0].trim())
|
|
.filter(t => t.length > 1 && t.length < 25)
|
|
.filter((t, i, arr) => arr.indexOf(t) === i)
|
|
.slice(0, 20);
|
|
});
|
|
console.log(C.red(` [NAV] L1 "${level1}" not found.`));
|
|
console.log(C.dim(` [NAV] Available: ${debugTexts.join(', ')}`));
|
|
return false;
|
|
}
|
|
await sleep(800); // Wait for accordion animation to expand
|
|
|
|
// Phase 3: Verify accordion expanded (L2 items should now be visible)
|
|
if (level2) {
|
|
// Wait for L2 items to appear after accordion expansion
|
|
let l2Visible = false;
|
|
for (let retryWait = 0; retryWait < 3; retryWait++) {
|
|
l2Visible = await page.evaluate(
|
|
(l2Text) => {
|
|
const items = document.querySelectorAll('a, button, [role="menuitem"], [role="treeitem"]');
|
|
return Array.from(items).some(el => {
|
|
const t = (el.innerText || '').trim();
|
|
return t === l2Text || t.includes(l2Text);
|
|
});
|
|
},
|
|
level2
|
|
);
|
|
if (l2Visible) break;
|
|
// Accordion might not have expanded - try clicking L1 again
|
|
if (retryWait === 1) {
|
|
await page.evaluate(
|
|
async ({ l1Text }) => {
|
|
const candidates = document.querySelectorAll('nav button, nav a, [role="menuitem"], [role="treeitem"]');
|
|
for (const el of candidates) {
|
|
const firstLine = (el.innerText || '').split('\n')[0].trim();
|
|
if (firstLine === l1Text || firstLine.startsWith(l1Text)) {
|
|
el.click();
|
|
break;
|
|
}
|
|
}
|
|
},
|
|
{ l1Text: level1 }
|
|
);
|
|
}
|
|
await sleep(600);
|
|
}
|
|
|
|
// Phase 4: Find and click L2 menu item
|
|
const l2Found = await page.evaluate(
|
|
async ({ l2Text }) => {
|
|
const sidebar = document.querySelector('.sidebar-scroll, [data-sidebar="content"], nav');
|
|
const maxAttempts = 15;
|
|
|
|
for (let i = 0; i < maxAttempts; i++) {
|
|
const items = Array.from(
|
|
document.querySelectorAll('a, button, [role="menuitem"], [role="treeitem"]')
|
|
);
|
|
|
|
// Exact match on innerText (most reliable)
|
|
let match = items.find(el => (el.innerText || '').trim() === l2Text);
|
|
|
|
// Partial match fallback
|
|
if (!match) {
|
|
match = items.find(el => {
|
|
const t = (el.innerText || '').trim();
|
|
return t.includes(l2Text) && t.length < l2Text.length + 15;
|
|
});
|
|
}
|
|
|
|
if (match) {
|
|
match.scrollIntoView({ behavior: 'instant', block: 'center' });
|
|
await new Promise(r => setTimeout(r, 150));
|
|
match.click();
|
|
return { found: true };
|
|
}
|
|
|
|
if (sidebar) {
|
|
sidebar.scrollBy({ top: 100, behavior: 'instant' });
|
|
await new Promise(r => setTimeout(r, 120));
|
|
}
|
|
}
|
|
return { found: false };
|
|
},
|
|
{ l2Text: level2 }
|
|
);
|
|
|
|
if (!l2Found.found) {
|
|
console.log(C.red(` [NAV] L2 "${level2}" not found under "${level1}".`));
|
|
return false;
|
|
}
|
|
await sleep(2000); // Wait for page load after L2 click
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* navigateViaMenu with automatic retry (up to 2 attempts).
|
|
*/
|
|
async function navigateViaMenuWithRetry(page, level1, level2, maxRetries = 2) {
|
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
const ok = await navigateViaMenu(page, level1, level2);
|
|
if (ok) return true;
|
|
|
|
if (attempt < maxRetries) {
|
|
console.log(C.yellow(` [NAV] Retry ${attempt}/${maxRetries - 1} for ${level1} > ${level2}`));
|
|
// Go back to dashboard and try again
|
|
try {
|
|
await page.goto(DASHBOARD_URL, { waitUntil: 'load', timeout: 12000 });
|
|
await sleep(1500);
|
|
await ensureSidebarExpanded(page);
|
|
await waitForSidebarReady(page, 6000);
|
|
} catch (e) {
|
|
await sleep(2000);
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// ─── Dashboard Navigation ───────────────────────────────────
|
|
|
|
async function ensureLoggedIn(page) {
|
|
const url = page.url();
|
|
if (url.includes('/login')) {
|
|
// Need to re-login
|
|
await page.fill('#userId', AUTH.username);
|
|
await page.fill('#password', AUTH.password);
|
|
await page.click("button[type='submit']");
|
|
await sleep(3000);
|
|
}
|
|
}
|
|
|
|
async function goToDashboard(page) {
|
|
// Force sidebar expanded state BEFORE navigation so page loads with full menu
|
|
try {
|
|
await page.evaluate(() => {
|
|
try {
|
|
const raw = localStorage.getItem('sam-menu');
|
|
if (raw) {
|
|
const data = JSON.parse(raw);
|
|
if (data.state) {
|
|
data.state.sidebarCollapsed = false;
|
|
localStorage.setItem('sam-menu', JSON.stringify(data));
|
|
}
|
|
}
|
|
} catch (e) {}
|
|
});
|
|
} catch (e) { /* page may not be ready yet */ }
|
|
|
|
// Attempt 1: Navigate to dashboard
|
|
try {
|
|
await page.goto(DASHBOARD_URL, { waitUntil: 'load', timeout: 15000 });
|
|
await sleep(1000);
|
|
await ensureLoggedIn(page);
|
|
await ensureSidebarExpanded(page);
|
|
await waitForSidebarReady(page, 8000);
|
|
return;
|
|
} catch (e) {
|
|
const dbgUrl = page.url();
|
|
console.log(C.yellow(` [DASH] attempt 1 failed. URL: ${dbgUrl}, err: ${e.message?.substring(0, 80)}`));
|
|
}
|
|
|
|
// Attempt 2: Reload to force fresh render
|
|
try {
|
|
await page.reload({ waitUntil: 'load', timeout: 10000 });
|
|
await sleep(1500);
|
|
await ensureLoggedIn(page);
|
|
await ensureSidebarExpanded(page);
|
|
await waitForSidebarReady(page, 8000);
|
|
return;
|
|
} catch (e) {
|
|
const dbgUrl = page.url();
|
|
console.log(C.yellow(` [DASH] reload failed. URL: ${dbgUrl}, err: ${e.message?.substring(0, 80)}`));
|
|
}
|
|
|
|
// Attempt 3: Full re-login
|
|
try {
|
|
await page.goto(`${BASE_URL}/ko/login`, { waitUntil: 'load', timeout: 10000 });
|
|
await sleep(500);
|
|
try {
|
|
await page.fill('#userId', AUTH.username);
|
|
await page.fill('#password', AUTH.password);
|
|
await page.click("button[type='submit']");
|
|
await sleep(3000);
|
|
} catch (loginErr) { /* may already be on dashboard */ }
|
|
await ensureSidebarExpanded(page);
|
|
await waitForSidebarReady(page, 8000);
|
|
} catch (e) {
|
|
await sleep(2000);
|
|
}
|
|
}
|
|
|
|
const SCENARIO_TIMEOUT = 180000; // 3 minutes per scenario (batch-create needs extra time)
|
|
const WORKFLOW_TIMEOUT = 300000; // 5 minutes for workflow scenarios (multi-module chains)
|
|
const PERFORMANCE_TIMEOUT = 120000; // 2 minutes for performance scenarios
|
|
|
|
/** Determine timeout based on scenario category */
|
|
function getScenarioTimeout(filename) {
|
|
if (filename.startsWith('workflow-')) return WORKFLOW_TIMEOUT;
|
|
if (filename.startsWith('perf-')) return PERFORMANCE_TIMEOUT;
|
|
return SCENARIO_TIMEOUT;
|
|
}
|
|
|
|
/** Classify scenario into a category for summary grouping */
|
|
function getScenarioCategory(filename) {
|
|
if (filename.startsWith('workflow-')) return 'workflow';
|
|
if (filename.startsWith('perf-')) return 'performance';
|
|
if (filename.startsWith('edge-')) return 'edge-case';
|
|
if (filename.startsWith('a11y-')) return 'accessibility';
|
|
return 'functional';
|
|
}
|
|
|
|
// ─── Page Health Verify ─────────────────────────────────────
|
|
|
|
/**
|
|
* Pre-flight page health check.
|
|
* Runs AFTER menu navigation, BEFORE step execution.
|
|
* Detects page crashes, console errors, Error Boundaries, API failures.
|
|
*
|
|
* Returns: { healthy: boolean, diagnosis: { ... } }
|
|
*/
|
|
async function verifyPageHealth(page, timeout = 8000) {
|
|
const diagnosis = {
|
|
healthy: true,
|
|
url: '',
|
|
crashed: false,
|
|
errorBoundary: null,
|
|
consoleErrors: [],
|
|
apiErrors: [],
|
|
emptySelectValues: 0,
|
|
blankPage: false,
|
|
loadTimeout: false,
|
|
};
|
|
|
|
try {
|
|
diagnosis.url = page.url();
|
|
|
|
// 1. Collect console errors captured during page load
|
|
// (we attach listener before navigation in runScenario, store them on page object)
|
|
diagnosis.consoleErrors = (page.__e2e_console_errors || []).slice(-10);
|
|
|
|
// 2. Check for Error Boundary / crash screen / blank page
|
|
const pageState = await Promise.race([
|
|
page.evaluate(() => {
|
|
const bodyText = document.body?.innerText || '';
|
|
|
|
// Error Boundary patterns (React)
|
|
// NOTE: bodyText(innerText)만 사용. innerHTML은 i18n 번역 JSON에
|
|
// "서버 오류가 발생했습니다" 등이 포함되어 false positive 발생함
|
|
const errorBoundaryPatterns = [
|
|
'오류가 발생했습니다', '일시적인 오류',
|
|
'Something went wrong', 'Error boundary',
|
|
'An error occurred', 'Unhandled Runtime Error',
|
|
'Application error',
|
|
];
|
|
const foundErrorText = errorBoundaryPatterns.find(p =>
|
|
bodyText.includes(p)
|
|
);
|
|
|
|
// Blank page detection
|
|
const hasContent = bodyText.trim().length > 50;
|
|
const hasMainContent = !!document.querySelector(
|
|
'table, [class*="content"], main, [role="main"], [class*="page"], [class*="list"]'
|
|
);
|
|
|
|
// Console errors (if captured by step-executor ApiMonitor)
|
|
const apiErrors = (window.__API_ERRORS__ || []).filter(e =>
|
|
e.status >= 400 || e.error
|
|
);
|
|
|
|
// Empty Select.Item values (the exact bug pattern)
|
|
const emptySelectItems = document.querySelectorAll(
|
|
'[role="option"][data-value=""], select option[value=""]'
|
|
);
|
|
|
|
// Check for React error overlay (dev mode)
|
|
const reactErrorOverlay = document.querySelector(
|
|
'[data-nextjs-dialog], #__next-build-error, [class*="nextjs-container-errors"]'
|
|
);
|
|
|
|
return {
|
|
foundErrorText,
|
|
hasContent,
|
|
hasMainContent,
|
|
apiErrorCount: apiErrors.length,
|
|
apiErrors: apiErrors.slice(0, 5).map(e => ({
|
|
url: (e.url || '').substring(0, 100),
|
|
status: e.status,
|
|
method: e.method,
|
|
error: e.error,
|
|
})),
|
|
emptySelectItems: emptySelectItems.length,
|
|
reactErrorOverlay: !!reactErrorOverlay,
|
|
reactErrorMsg: reactErrorOverlay?.innerText?.substring(0, 200) || null,
|
|
};
|
|
}),
|
|
sleep(timeout).then(() => ({ timeout: true })),
|
|
]);
|
|
|
|
if (pageState.timeout) {
|
|
diagnosis.healthy = false;
|
|
diagnosis.loadTimeout = true;
|
|
return diagnosis;
|
|
}
|
|
|
|
// Error Boundary detected
|
|
if (pageState.foundErrorText) {
|
|
diagnosis.healthy = false;
|
|
diagnosis.crashed = true;
|
|
diagnosis.errorBoundary = pageState.foundErrorText;
|
|
}
|
|
|
|
// React error overlay (dev/next.js)
|
|
if (pageState.reactErrorOverlay) {
|
|
diagnosis.healthy = false;
|
|
diagnosis.crashed = true;
|
|
diagnosis.errorBoundary = pageState.reactErrorMsg || 'React Error Overlay detected';
|
|
}
|
|
|
|
// Blank page
|
|
if (!pageState.hasContent && !pageState.hasMainContent) {
|
|
diagnosis.healthy = false;
|
|
diagnosis.blankPage = true;
|
|
}
|
|
|
|
// API errors
|
|
if (pageState.apiErrorCount > 0) {
|
|
diagnosis.apiErrors = pageState.apiErrors;
|
|
// API 500 errors make the page unhealthy
|
|
if (pageState.apiErrors.some(e => e.status >= 500)) {
|
|
diagnosis.healthy = false;
|
|
}
|
|
}
|
|
|
|
// Empty Select values (Radix UI crash pattern)
|
|
diagnosis.emptySelectValues = pageState.emptySelectItems;
|
|
if (pageState.emptySelectItems > 0) {
|
|
diagnosis.healthy = false;
|
|
}
|
|
|
|
// Console errors make it unhealthy if they contain crash indicators
|
|
const criticalConsolePatterns = [
|
|
'Select.Item', 'must have a value', 'Uncaught', 'ChunkLoadError',
|
|
'Cannot read properties of null', 'Cannot read properties of undefined',
|
|
'Maximum update depth exceeded', 'Minified React error',
|
|
];
|
|
const criticalConsoleErrors = diagnosis.consoleErrors.filter(msg =>
|
|
criticalConsolePatterns.some(p => msg.includes(p))
|
|
);
|
|
if (criticalConsoleErrors.length > 0) {
|
|
diagnosis.healthy = false;
|
|
}
|
|
} catch (err) {
|
|
diagnosis.healthy = false;
|
|
diagnosis.crashed = true;
|
|
diagnosis.errorBoundary = `Health check error: ${err.message}`;
|
|
}
|
|
|
|
return diagnosis;
|
|
}
|
|
|
|
/**
|
|
* Post-failure diagnosis.
|
|
* Runs AFTER a scenario fails to collect detailed root cause information.
|
|
* Captures: console errors, DOM state, API logs, screenshot.
|
|
*
|
|
* Returns: { rootCause: string, details: { ... } }
|
|
*/
|
|
async function diagnoseFail(page, result, scenarioId) {
|
|
const diag = {
|
|
rootCause: 'unknown',
|
|
consoleErrors: [],
|
|
apiErrors: [],
|
|
pageState: null,
|
|
screenshotPath: null,
|
|
recommendations: [],
|
|
};
|
|
|
|
try {
|
|
// 1. Capture screenshot
|
|
const ssName = `diag_${scenarioId}_${getTimestamp()}.png`;
|
|
const ssPath = path.join(SCREENSHOTS_DIR, ssName);
|
|
try {
|
|
await page.screenshot({ path: ssPath, fullPage: false, timeout: 5000 });
|
|
diag.screenshotPath = ssPath;
|
|
} catch (e) { /* screenshot failed, continue */ }
|
|
|
|
// 2. Collect console errors
|
|
diag.consoleErrors = (page.__e2e_console_errors || []).slice(-20);
|
|
|
|
// 3. Collect page state & API errors
|
|
try {
|
|
const state = await Promise.race([
|
|
page.evaluate(() => {
|
|
const bodyText = document.body?.innerText || '';
|
|
|
|
// Error Boundary check
|
|
const errorPatterns = [
|
|
'오류가 발생했습니다', '일시적인 오류',
|
|
'Something went wrong', 'Error boundary',
|
|
];
|
|
const errorBoundary = errorPatterns.find(p => bodyText.includes(p));
|
|
|
|
// API errors from step-executor monitor
|
|
const apiLogs = window.__API_LOGS__ || [];
|
|
const apiErrors = (window.__API_ERRORS__ || []).slice(0, 10).map(e => ({
|
|
url: (e.url || '').substring(0, 120),
|
|
status: e.status,
|
|
method: e.method,
|
|
error: e.error,
|
|
}));
|
|
|
|
// Null data detection in rendered content
|
|
const nullPatterns = bodyText.match(/null|undefined/gi) || [];
|
|
|
|
// DOM stats
|
|
const domNodes = document.getElementsByTagName('*').length;
|
|
const tables = document.querySelectorAll('table');
|
|
const tableRowCount = tables.length > 0 ? tables[0].querySelectorAll('tbody tr').length : 0;
|
|
const hasLoadingSpinner = !!document.querySelector(
|
|
'.loading, .spinner, [class*="skeleton"], [class*="loading"], [class*="Skeleton"]'
|
|
);
|
|
|
|
return {
|
|
url: window.location.href,
|
|
errorBoundary,
|
|
apiTotal: apiLogs.length,
|
|
apiErrors,
|
|
nullCount: nullPatterns.length,
|
|
domNodes,
|
|
tableRowCount,
|
|
hasLoadingSpinner,
|
|
visibleText: bodyText.substring(0, 300),
|
|
};
|
|
}),
|
|
sleep(5000).then(() => null),
|
|
]);
|
|
|
|
if (state) {
|
|
diag.pageState = state;
|
|
diag.apiErrors = state.apiErrors;
|
|
|
|
// Classify root cause
|
|
if (state.errorBoundary) {
|
|
diag.rootCause = 'page_crash';
|
|
diag.recommendations.push('페이지 크래시 - Error Boundary 활성화됨. Console 에러 확인 필요');
|
|
} else if (state.apiErrors.some(e => e.status >= 500)) {
|
|
diag.rootCause = 'api_server_error';
|
|
diag.recommendations.push('백엔드 서버 에러 (5xx). 서버 로그 확인 필요');
|
|
} else if (state.apiErrors.some(e => e.status === 401 || e.status === 403)) {
|
|
diag.rootCause = 'auth_error';
|
|
diag.recommendations.push('인증/권한 에러. 세션 만료 가능성');
|
|
} else if (state.hasLoadingSpinner) {
|
|
diag.rootCause = 'infinite_loading';
|
|
diag.recommendations.push('무한 로딩 상태. API 미응답 또는 프론트엔드 상태 관리 버그');
|
|
} else if (state.tableRowCount === 0 && state.apiTotal > 0) {
|
|
diag.rootCause = 'empty_data';
|
|
diag.recommendations.push('API 응답은 있으나 테이블 데이터 없음. 데이터 변환 또는 필터 문제');
|
|
}
|
|
}
|
|
} catch (e) {
|
|
diag.rootCause = 'page_unresponsive';
|
|
diag.recommendations.push('페이지 응답 없음 (evaluate 실패). 네비게이션 에러 또는 크래시');
|
|
}
|
|
|
|
// 4. Console error pattern matching for specific root causes
|
|
const consoleText = diag.consoleErrors.join(' ');
|
|
if (consoleText.includes('Select.Item') || consoleText.includes('must have a value')) {
|
|
diag.rootCause = 'select_empty_value';
|
|
diag.recommendations = ['Radix Select.Item에 빈 value 전달됨. 데이터 transform에서 null/빈값 방어 필요'];
|
|
} else if (consoleText.includes('ChunkLoadError') || consoleText.includes('Loading chunk')) {
|
|
diag.rootCause = 'chunk_load_error';
|
|
diag.recommendations = ['JS 번들 로드 실패. 배포 상태 또는 네트워크 문제'];
|
|
} else if (consoleText.includes('Cannot read properties of null') || consoleText.includes('Cannot read properties of undefined')) {
|
|
diag.rootCause = 'null_reference';
|
|
diag.recommendations = ['Null 참조 에러. API 응답에서 예상치 못한 null 데이터 가능성'];
|
|
} else if (consoleText.includes('Maximum update depth')) {
|
|
diag.rootCause = 'infinite_render_loop';
|
|
diag.recommendations = ['React 무한 렌더 루프. useEffect 의존성 배열 또는 상태 업데이트 로직 확인'];
|
|
}
|
|
|
|
// 5. Check failed step patterns for additional context
|
|
if (result.steps) {
|
|
const failedSteps = result.steps.filter(s => s.status === 'fail');
|
|
const timeoutSteps = failedSteps.filter(s =>
|
|
(s.error || s.details || '').includes('timeout') || (s.error || s.details || '').includes('Timeout')
|
|
);
|
|
if (timeoutSteps.length > 0 && diag.rootCause === 'unknown') {
|
|
diag.rootCause = 'element_timeout';
|
|
diag.recommendations.push('요소 대기 타임아웃. 페이지 로드 지연 또는 셀렉터 불일치');
|
|
}
|
|
}
|
|
|
|
} catch (err) {
|
|
diag.rootCause = 'diagnosis_error';
|
|
diag.recommendations.push(`진단 중 에러: ${err.message}`);
|
|
}
|
|
|
|
return diag;
|
|
}
|
|
|
|
// ─── Scenario Runner ────────────────────────────────────────
|
|
|
|
async function runScenario(page, scenarioPath) {
|
|
const scenarioJson = JSON.parse(fs.readFileSync(scenarioPath, 'utf-8'));
|
|
const { id, name, steps, selectors, menuNavigation } = scenarioJson;
|
|
|
|
const result = {
|
|
id: id || path.basename(scenarioPath, '.json'),
|
|
name: name || id,
|
|
steps: [],
|
|
passed: 0,
|
|
failed: 0,
|
|
warned: 0,
|
|
totalSteps: 0,
|
|
apiSummary: null,
|
|
error: null,
|
|
stoppedReason: 'complete',
|
|
currentUrl: '',
|
|
startTime: Date.now(),
|
|
endTime: 0,
|
|
healthCheck: null, // Page health verification result
|
|
diagnosis: null, // Post-failure diagnosis result
|
|
};
|
|
|
|
// Attach console error listener for this scenario
|
|
page.__e2e_console_errors = [];
|
|
const consoleHandler = (msg) => {
|
|
if (msg.type() === 'error') {
|
|
page.__e2e_console_errors.push(msg.text().substring(0, 500));
|
|
}
|
|
};
|
|
page.on('console', consoleHandler);
|
|
|
|
// Also capture uncaught page errors
|
|
const pageErrorHandler = (err) => {
|
|
page.__e2e_console_errors.push(`[PAGE_ERROR] ${err.message}`.substring(0, 500));
|
|
};
|
|
page.on('pageerror', pageErrorHandler);
|
|
|
|
if (!steps || steps.length === 0) {
|
|
result.error = 'No steps defined';
|
|
result.stoppedReason = 'no_steps';
|
|
result.endTime = Date.now();
|
|
page.removeListener('console', consoleHandler);
|
|
page.removeListener('pageerror', pageErrorHandler);
|
|
return result;
|
|
}
|
|
|
|
try {
|
|
// Navigate to dashboard first
|
|
await goToDashboard(page);
|
|
|
|
// Inject executor
|
|
await injectExecutor(page);
|
|
|
|
// Menu navigation if specified
|
|
if (menuNavigation && menuNavigation.level1) {
|
|
const navOk = await navigateViaMenuWithRetry(page, menuNavigation.level1, menuNavigation.level2);
|
|
if (!navOk) {
|
|
result.error = `Menu navigation failed: ${menuNavigation.level1} > ${menuNavigation.level2}`;
|
|
result.stoppedReason = 'navigation_failed';
|
|
result.endTime = Date.now();
|
|
page.removeListener('console', consoleHandler);
|
|
page.removeListener('pageerror', pageErrorHandler);
|
|
return result;
|
|
}
|
|
// Re-inject after navigation
|
|
await sleep(1000);
|
|
await injectExecutor(page);
|
|
}
|
|
|
|
// ─── Page Health Check (Pre-flight) ───────────────────
|
|
const health = await verifyPageHealth(page);
|
|
result.healthCheck = health;
|
|
|
|
if (!health.healthy) {
|
|
// Page is unhealthy - run diagnosis immediately and abort
|
|
const diag = await diagnoseFail(page, result, result.id);
|
|
result.diagnosis = diag;
|
|
|
|
// Build descriptive error message
|
|
const reasons = [];
|
|
if (health.crashed || health.errorBoundary)
|
|
reasons.push(`페이지 크래시: ${health.errorBoundary}`);
|
|
if (health.blankPage)
|
|
reasons.push('빈 페이지 (콘텐츠 없음)');
|
|
if (health.emptySelectValues > 0)
|
|
reasons.push(`빈 Select 값 ${health.emptySelectValues}개 감지`);
|
|
if (health.loadTimeout)
|
|
reasons.push('페이지 로드 타임아웃');
|
|
if (health.apiErrors.length > 0)
|
|
reasons.push(`API 에러 ${health.apiErrors.length}건 (${health.apiErrors.map(e => `${e.status} ${e.method}`).join(', ')})`);
|
|
|
|
const consoleSnippet = (health.consoleErrors || [])
|
|
.filter(msg => msg.length > 10)
|
|
.slice(0, 3)
|
|
.map(msg => msg.substring(0, 150));
|
|
|
|
result.error = `[HEALTH_CHECK] ${reasons.join(' | ')}`;
|
|
if (consoleSnippet.length > 0) {
|
|
result.error += ` | Console: ${consoleSnippet.join('; ')}`;
|
|
}
|
|
result.stoppedReason = 'health_check_failed';
|
|
|
|
console.log(C.yellow(` [HEALTH] ✘ ${reasons[0] || 'unhealthy'}`));
|
|
if (diag.rootCause !== 'unknown') {
|
|
console.log(C.yellow(` [DIAG] root cause: ${diag.rootCause}`));
|
|
}
|
|
|
|
result.endTime = Date.now();
|
|
page.removeListener('console', consoleHandler);
|
|
page.removeListener('pageerror', pageErrorHandler);
|
|
return result;
|
|
}
|
|
|
|
// Run steps in batches (handling navigation stops)
|
|
let currentIndex = 0;
|
|
let vars = {};
|
|
const allResults = [];
|
|
|
|
while (currentIndex < steps.length) {
|
|
const batch = steps.slice(currentIndex);
|
|
|
|
let batchResult;
|
|
try {
|
|
batchResult = await page.evaluate(
|
|
async ({ batch, vars, selectors }) => {
|
|
const r = await window.__E2E__.runBatch(batch, vars, { selectors: selectors || {} });
|
|
return r;
|
|
},
|
|
{ batch, vars, selectors: selectors || {} }
|
|
);
|
|
} catch (evalErr) {
|
|
// If evaluate fails (page navigated away, etc.), try re-injection
|
|
try {
|
|
await sleep(2000);
|
|
await injectExecutor(page);
|
|
batchResult = await page.evaluate(
|
|
async ({ batch, vars, selectors }) => {
|
|
const r = await window.__E2E__.runBatch(batch, vars, { selectors: selectors || {} });
|
|
return r;
|
|
},
|
|
{ batch, vars, selectors: selectors || {} }
|
|
);
|
|
} catch (retryErr) {
|
|
result.error = `Evaluate failed: ${retryErr.message}`;
|
|
result.stoppedReason = 'evaluate_error';
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Collect results
|
|
if (batchResult.results) {
|
|
allResults.push(...batchResult.results);
|
|
}
|
|
vars = batchResult.variables || vars;
|
|
result.apiSummary = batchResult.apiSummary;
|
|
result.currentUrl = batchResult.currentUrl;
|
|
|
|
// Handle stoppedReason
|
|
if (batchResult.stoppedReason === 'complete') {
|
|
break;
|
|
}
|
|
|
|
if (batchResult.stoppedReason === 'navigation') {
|
|
await sleep(2000);
|
|
try {
|
|
await injectExecutor(page);
|
|
} catch (e) {
|
|
// Page might still be loading
|
|
await sleep(3000);
|
|
try {
|
|
await injectExecutor(page);
|
|
} catch (e2) {
|
|
result.error = `Re-injection failed after navigation: ${e2.message}`;
|
|
result.stoppedReason = 'reinjection_failed';
|
|
break;
|
|
}
|
|
}
|
|
currentIndex += batchResult.stoppedAtIndex;
|
|
continue;
|
|
}
|
|
|
|
if (batchResult.stoppedReason === 'native_required') {
|
|
// Take screenshot
|
|
const ssName = `${result.id}_step${batchResult.stoppedAtIndex}_${getTimestamp()}.png`;
|
|
try {
|
|
await page.screenshot({ path: path.join(SCREENSHOTS_DIR, ssName), fullPage: false });
|
|
} catch (ssErr) {
|
|
// Screenshot failed, continue
|
|
}
|
|
currentIndex += batchResult.stoppedAtIndex + 1;
|
|
// Re-inject in case page changed
|
|
try { await injectExecutor(page); } catch (e) { /* ignore */ }
|
|
continue;
|
|
}
|
|
|
|
if (batchResult.stoppedReason === 'critical_failure') {
|
|
result.stoppedReason = 'critical_failure';
|
|
break;
|
|
}
|
|
|
|
// Unknown stoppedReason
|
|
break;
|
|
}
|
|
|
|
// Aggregate results
|
|
result.steps = allResults;
|
|
result.totalSteps = allResults.length;
|
|
result.passed = allResults.filter((r) => r.status === 'pass').length;
|
|
result.failed = allResults.filter((r) => r.status === 'fail').length;
|
|
result.warned = allResults.filter((r) => r.status === 'warn').length;
|
|
if (result.stoppedReason === 'complete' && result.failed > 0) {
|
|
result.stoppedReason = 'completed_with_failures';
|
|
}
|
|
|
|
// ─── Post-failure Diagnosis ────────────────────────────
|
|
if (result.failed > 0 || result.error) {
|
|
try {
|
|
const diag = await diagnoseFail(page, result, result.id);
|
|
result.diagnosis = diag;
|
|
if (diag.rootCause !== 'unknown') {
|
|
console.log(C.yellow(` [DIAG] root cause: ${diag.rootCause}`));
|
|
}
|
|
} catch (diagErr) {
|
|
// Diagnosis itself failed, don't block
|
|
}
|
|
}
|
|
} catch (err) {
|
|
result.error = err.message;
|
|
result.stoppedReason = 'exception';
|
|
// Try diagnosis even on exception
|
|
try {
|
|
const diag = await diagnoseFail(page, result, result.id);
|
|
result.diagnosis = diag;
|
|
} catch (diagErr) { /* ignore */ }
|
|
}
|
|
|
|
// Cleanup event listeners
|
|
page.removeListener('console', consoleHandler);
|
|
page.removeListener('pageerror', pageErrorHandler);
|
|
|
|
result.endTime = Date.now();
|
|
return result;
|
|
}
|
|
|
|
// ─── Report Generation ──────────────────────────────────────
|
|
|
|
function generateReport(result, timestamp) {
|
|
const duration = ((result.endTime - result.startTime) / 1000).toFixed(1);
|
|
const hasFail = result.failed > 0 || result.error;
|
|
const status = hasFail ? 'FAIL' : 'PASS';
|
|
const icon = hasFail ? '❌' : '✅';
|
|
|
|
const stepsTable = result.steps
|
|
.map((s) => {
|
|
const statusIcon = s.status === 'pass' ? '✅' : s.status === 'fail' ? '❌' : '⚠️';
|
|
const phase = s.phase || '-';
|
|
const details = (s.details || '').substring(0, 80).replace(/\|/g, '/');
|
|
return `| ${s.stepId} | ${s.name} | ${phase} | ${statusIcon} | ${s.duration}ms | ${details} |`;
|
|
})
|
|
.join('\n');
|
|
|
|
const failedSteps = result.steps.filter((s) => s.status === 'fail');
|
|
const failedTable = failedSteps.length > 0
|
|
? failedSteps
|
|
.map((s) => `| ${s.stepId} | ${s.name} | ${s.phase || '-'} | ${(s.error || s.details || '').substring(0, 100)} |`)
|
|
.join('\n')
|
|
: '';
|
|
|
|
const api = result.apiSummary || { total: 0, success: 0, failed: 0, avgResponseTime: 0, slowCalls: 0 };
|
|
|
|
let md = `# ${icon} E2E 테스트 ${hasFail ? '실패' : '성공'}: ${result.name}
|
|
|
|
**테스트 ID**: ${result.id} | **실행**: ${timestamp} | **결과**: ${status}
|
|
**소요 시간**: ${duration}초${result.error ? ` | **에러**: ${result.error}` : ''}${result.stoppedReason !== 'complete' && result.stoppedReason !== 'completed_with_failures' ? ` | **중단 사유**: ${result.stoppedReason}` : ''}
|
|
|
|
## 테스트 요약
|
|
| 전체 | 성공 | 실패 | 경고 | 성공률 |
|
|
|------|------|------|------|--------|
|
|
| ${result.totalSteps} | ${result.passed} | ${result.failed} | ${result.warned} | ${result.totalSteps > 0 ? Math.round((result.passed / result.totalSteps) * 100) : 0}% |
|
|
`;
|
|
|
|
if (failedTable) {
|
|
md += `
|
|
## 실패 스텝
|
|
| # | 스텝 | Phase | 에러 |
|
|
|---|------|-------|------|
|
|
${failedTable}
|
|
`;
|
|
}
|
|
|
|
md += `
|
|
## 전체 스텝 결과
|
|
| # | 스텝 | Phase | 상태 | 소요시간 | 비고 |
|
|
|---|------|-------|------|---------|------|
|
|
${stepsTable || '| - | (스텝 없음) | - | - | - | - |'}
|
|
|
|
## API 요약
|
|
| 총 호출 | 성공 | 실패 | 평균 응답 | 느린 호출(>2s) |
|
|
|---------|------|------|----------|--------------|
|
|
| ${api.total} | ${api.success} | ${api.failed} | ${api.avgResponseTime}ms | ${api.slowCalls} |
|
|
`;
|
|
|
|
// Health Check section
|
|
if (result.healthCheck) {
|
|
const h = result.healthCheck;
|
|
md += '\n## 페이지 건강 검사\n';
|
|
md += '| 항목 | 결과 |\n|------|------|\n';
|
|
md += '| 상태 | ' + (h.healthy ? '✅ 정상' : '❌ 비정상') + ' |\n';
|
|
md += '| URL | ' + (h.url || '-') + ' |\n';
|
|
if (h.crashed) md += '| 크래시 | ' + (h.errorBoundary || 'Yes') + ' |\n';
|
|
if (h.blankPage) md += '| 빈 페이지 | Yes |\n';
|
|
if (h.loadTimeout) md += '| 로드 타임아웃 | Yes |\n';
|
|
if (h.emptySelectValues > 0) md += '| 빈 Select 값 | ' + h.emptySelectValues + '개 |\n';
|
|
if (h.apiErrors && h.apiErrors.length > 0) {
|
|
const apiErrStr = h.apiErrors.map(function(e) { return e.status + ' ' + e.method + ' ' + e.url; }).join(', ');
|
|
md += '| API 에러 | ' + apiErrStr + ' |\n';
|
|
}
|
|
|
|
if (h.consoleErrors && h.consoleErrors.length > 0) {
|
|
md += '\n### 콘솔 에러 (Health Check)\n';
|
|
h.consoleErrors.slice(0, 5).forEach(function(err, i) {
|
|
md += (i + 1) + '. `' + err.substring(0, 200) + '`\n';
|
|
});
|
|
}
|
|
}
|
|
|
|
// Diagnosis section
|
|
if (result.diagnosis) {
|
|
const d = result.diagnosis;
|
|
md += '\n## 자동 진단\n';
|
|
md += '| 항목 | 내용 |\n|------|------|\n';
|
|
md += '| 근본 원인 | **' + d.rootCause + '** |\n';
|
|
if (d.screenshotPath) md += '| 스크린샷 | ' + path.basename(d.screenshotPath) + ' |\n';
|
|
|
|
if (d.recommendations && d.recommendations.length > 0) {
|
|
md += '\n### 권장 조치\n';
|
|
d.recommendations.forEach(function(rec, i) {
|
|
md += (i + 1) + '. ' + rec + '\n';
|
|
});
|
|
}
|
|
|
|
if (d.consoleErrors && d.consoleErrors.length > 0) {
|
|
md += '\n### 콘솔 에러 (진단)\n';
|
|
d.consoleErrors.slice(0, 10).forEach(function(err, i) {
|
|
md += (i + 1) + '. `' + err.substring(0, 200) + '`\n';
|
|
});
|
|
}
|
|
|
|
if (d.pageState) {
|
|
const ps = d.pageState;
|
|
md += '\n### 페이지 상태\n';
|
|
md += '| 항목 | 값 |\n|------|----|\n';
|
|
md += '| DOM 노드 | ' + (ps.domNodes || '-') + ' |\n';
|
|
md += '| 테이블 행 | ' + (ps.tableRowCount || 0) + ' |\n';
|
|
md += '| API 호출 수 | ' + (ps.apiTotal || 0) + ' |\n';
|
|
md += '| 로딩 스피너 | ' + (ps.hasLoadingSpinner ? 'Yes' : 'No') + ' |\n';
|
|
if (ps.errorBoundary) md += '| Error Boundary | ' + ps.errorBoundary + ' |\n';
|
|
}
|
|
}
|
|
|
|
return md;
|
|
}
|
|
|
|
function saveReport(result, timestamp) {
|
|
const hasFail = result.failed > 0 || result.error;
|
|
|
|
if (hasFail) {
|
|
const filePath = path.join(RESULTS_DIR, `Fail-${result.id}_${timestamp}.md`);
|
|
fs.writeFileSync(filePath, generateReport(result, timestamp), 'utf-8');
|
|
return filePath;
|
|
} else {
|
|
const filePath = path.join(SUCCESS_DIR, `OK-${result.id}_${timestamp}.md`);
|
|
fs.writeFileSync(filePath, generateReport(result, timestamp), 'utf-8');
|
|
return filePath;
|
|
}
|
|
}
|
|
|
|
// ─── Summary Report ─────────────────────────────────────────
|
|
|
|
function generateSummaryReport(allResults, totalTime, timestamp) {
|
|
const passed = allResults.filter((r) => !r.error && r.failed === 0).length;
|
|
const failed = allResults.length - passed;
|
|
|
|
// Categorize results
|
|
const categories = {};
|
|
allResults.forEach((r) => {
|
|
const cat = getScenarioCategory(r.id || '');
|
|
if (!categories[cat]) categories[cat] = [];
|
|
categories[cat].push(r);
|
|
});
|
|
|
|
let md = `# E2E 전체 테스트 결과 요약
|
|
|
|
**실행 시간**: ${timestamp}
|
|
**총 소요 시간**: ${(totalTime / 1000 / 60).toFixed(1)}분
|
|
**전체 시나리오**: ${allResults.length}개 | **성공**: ${passed}개 | **실패**: ${failed}개
|
|
|
|
## 카테고리별 요약
|
|
| 카테고리 | 시나리오 수 | 성공 | 실패 | 성공률 |
|
|
|---------|-----------|------|------|--------|
|
|
`;
|
|
|
|
const catNames = { functional: '기능 테스트', workflow: '비즈니스 워크플로우', performance: '성능 테스트', 'edge-case': '엣지 케이스', accessibility: '접근성 검사' };
|
|
for (const [cat, results] of Object.entries(categories)) {
|
|
const catPassed = results.filter(r => !r.error && r.failed === 0).length;
|
|
const catFailed = results.length - catPassed;
|
|
const rate = results.length > 0 ? Math.round((catPassed / results.length) * 100) : 0;
|
|
md += `| ${catNames[cat] || cat} | ${results.length} | ${catPassed} | ${catFailed} | ${rate}% |\n`;
|
|
}
|
|
|
|
md += `
|
|
## 시나리오별 결과
|
|
| # | 시나리오 | 결과 | 스텝 | 성공 | 실패 | 소요(초) |
|
|
|---|---------|------|------|------|------|---------|
|
|
`;
|
|
|
|
allResults.forEach((r, i) => {
|
|
const hasFail = r.failed > 0 || r.error;
|
|
const icon = hasFail ? '❌' : '✅';
|
|
const duration = ((r.endTime - r.startTime) / 1000).toFixed(1);
|
|
md += `| ${i + 1} | ${r.name} | ${icon} | ${r.totalSteps} | ${r.passed} | ${r.failed} | ${duration} |\n`;
|
|
});
|
|
|
|
// Workflow summary section
|
|
if (categories.workflow && categories.workflow.length > 0) {
|
|
md += `\n## 비즈니스 워크플로우 상세\n`;
|
|
categories.workflow.forEach((r) => {
|
|
const hasFail = r.failed > 0 || r.error;
|
|
const icon = hasFail ? '❌' : '✅';
|
|
const duration = ((r.endTime - r.startTime) / 1000).toFixed(1);
|
|
md += `\n### ${icon} ${r.name}\n`;
|
|
md += `- 스텝: ${r.passed}/${r.totalSteps} 성공 | 소요: ${duration}초\n`;
|
|
if (r.error) md += `- 에러: ${r.error}\n`;
|
|
const phases = r.steps.filter(s => s.phase).map(s => `${s.phase}(${s.status === 'pass' ? '✅' : '❌'})`);
|
|
if (phases.length > 0) md += `- 단계: ${phases.join(' → ')}\n`;
|
|
});
|
|
}
|
|
|
|
// Performance summary section
|
|
if (categories.performance && categories.performance.length > 0) {
|
|
md += `\n## 성능 테스트 요약\n`;
|
|
md += `| 페이지 | 로드 시간 | 등급 | API 평균 | DOM 노드 |\n`;
|
|
md += `|--------|----------|------|---------|----------|\n`;
|
|
categories.performance.forEach((r) => {
|
|
const perfStep = r.steps.find(s => s.phase === 'PERF_MEASURE');
|
|
const perfData = perfStep?.details ? (() => { try { return JSON.parse(perfStep.details); } catch(e) { return null; } })() : null;
|
|
if (perfData) {
|
|
md += `| ${r.name} | ${perfData.loadTime || '-'}ms | ${perfData.grade || '-'} | ${perfData.apiAvg || '-'}ms | ${perfData.domNodes || '-'} |\n`;
|
|
} else {
|
|
md += `| ${r.name} | - | - | - | - |\n`;
|
|
}
|
|
});
|
|
}
|
|
|
|
// Accessibility summary section
|
|
if (categories.accessibility && categories.accessibility.length > 0) {
|
|
md += `\n## 접근성 검사 요약\n`;
|
|
md += `| 페이지 | 점수 | 등급 | Critical | Serious | Moderate |\n`;
|
|
md += `|--------|------|------|----------|---------|----------|\n`;
|
|
categories.accessibility.forEach((r) => {
|
|
const a11yStep = r.steps.find(s => s.phase === 'A11Y_AUDIT');
|
|
const a11yData = a11yStep?.details ? (() => { try { return JSON.parse(a11yStep.details); } catch(e) { return null; } })() : null;
|
|
if (a11yData) {
|
|
md += `| ${r.name} | ${a11yData.score || '-'} | ${a11yData.grade || '-'} | ${a11yData.critical || 0} | ${a11yData.serious || 0} | ${a11yData.moderate || 0} |\n`;
|
|
} else {
|
|
md += `| ${r.name} | - | - | - | - | - |\n`;
|
|
}
|
|
});
|
|
}
|
|
|
|
if (failed > 0) {
|
|
md += `\n## 실패 시나리오 상세\n`;
|
|
allResults
|
|
.filter((r) => r.failed > 0 || r.error)
|
|
.forEach((r) => {
|
|
md += `\n### ❌ ${r.name} (${r.id})\n`;
|
|
if (r.error) md += `- **에러**: ${r.error}\n`;
|
|
|
|
// Diagnosis info
|
|
if (r.diagnosis && r.diagnosis.rootCause !== 'unknown') {
|
|
md += `- **진단**: ${r.diagnosis.rootCause}`;
|
|
if (r.diagnosis.recommendations && r.diagnosis.recommendations.length > 0) {
|
|
md += ` → ${r.diagnosis.recommendations[0]}`;
|
|
}
|
|
md += `\n`;
|
|
}
|
|
|
|
// Health check info
|
|
if (r.healthCheck && !r.healthCheck.healthy) {
|
|
const h = r.healthCheck;
|
|
const issues = [];
|
|
if (h.crashed) issues.push('크래시');
|
|
if (h.blankPage) issues.push('빈 페이지');
|
|
if (h.loadTimeout) issues.push('로드 타임아웃');
|
|
if (h.emptySelectValues > 0) issues.push(`빈 Select 값 ${h.emptySelectValues}개`);
|
|
if (issues.length > 0) md += `- **건강 검사**: ${issues.join(', ')}\n`;
|
|
}
|
|
|
|
const failSteps = r.steps.filter((s) => s.status === 'fail');
|
|
failSteps.forEach((s) => {
|
|
md += `- Step ${s.stepId} (${s.name}): ${s.error || s.details}\n`;
|
|
});
|
|
});
|
|
}
|
|
|
|
return md;
|
|
}
|
|
|
|
// ─── Main ───────────────────────────────────────────────────
|
|
|
|
async function main() {
|
|
console.log(C.bold('\n=== E2E 전체 테스트 러너 ==='));
|
|
console.log(`서버: ${BASE_URL}`);
|
|
console.log(`모드: ${HEADLESS ? 'headless' : 'headed'}`);
|
|
if (WORKFLOW_ONLY) console.log(`카테고리: workflow only`);
|
|
if (FILTER) console.log(`필터: ${FILTER}`);
|
|
if (EXCLUDE) console.log(`제외: ${EXCLUDE}`);
|
|
if (SKIP_PASSED) console.log(`스킵: 이미 성공한 시나리오 건너뛰기 (--skip-passed)`);
|
|
if (ITERATE) console.log(`반복 모드: 실패 시나리오 자동 재실행 (최대 ${MAX_ITERATIONS}회)`);
|
|
if (STAGE_MODE) console.log(`단계 모드: 카테고리별 순차 실행`);
|
|
if (FAIL_ONLY) console.log(`실패 집중 모드: 이전 실패 시나리오만 실행 (--fail-only)`);
|
|
console.log('');
|
|
|
|
// Ensure directories
|
|
ensureDir(RESULTS_DIR);
|
|
ensureDir(SUCCESS_DIR);
|
|
ensureDir(SCREENSHOTS_DIR);
|
|
|
|
// Collect scenario files (skip disabled scenarios)
|
|
let scenarioFiles = fs.readdirSync(SCENARIOS_DIR)
|
|
.filter((f) => f.endsWith('.json') && !f.startsWith('_'))
|
|
.filter((f) => {
|
|
try {
|
|
const data = JSON.parse(fs.readFileSync(path.join(SCENARIOS_DIR, f), 'utf-8'));
|
|
return data.enabled !== false; // Skip if explicitly disabled
|
|
} catch (e) {
|
|
return true; // Include if can't parse (will fail later with better error)
|
|
}
|
|
})
|
|
.sort();
|
|
|
|
if (WORKFLOW_ONLY) {
|
|
scenarioFiles = scenarioFiles.filter((f) => f.startsWith('workflow-'));
|
|
}
|
|
if (FILTER) {
|
|
scenarioFiles = scenarioFiles.filter((f) => f.includes(FILTER));
|
|
}
|
|
if (EXCLUDE) {
|
|
scenarioFiles = scenarioFiles.filter((f) => !f.includes(EXCLUDE));
|
|
}
|
|
|
|
// --skip-passed: 이미 성공한 시나리오 건너뛰기
|
|
let skippedCount = 0;
|
|
if (SKIP_PASSED) {
|
|
const passedIds = getPassedScenarioIds();
|
|
const before = scenarioFiles.length;
|
|
scenarioFiles = scenarioFiles.filter((f) => {
|
|
const id = f.replace('.json', '');
|
|
return !passedIds.has(id);
|
|
});
|
|
skippedCount = before - scenarioFiles.length;
|
|
if (skippedCount > 0) {
|
|
console.log(C.cyan(`이미 성공한 시나리오 ${skippedCount}개 건너뜀 (--skip-passed)\n`));
|
|
}
|
|
}
|
|
|
|
// --fail-only: 이전 실패 시나리오만 실행
|
|
if (FAIL_ONLY) {
|
|
const { ids: failedIds, source } = getFailedScenarioIds();
|
|
if (failedIds.size === 0) {
|
|
console.log(C.green('이전 실패 시나리오 없음! 전체 PASS 상태입니다.'));
|
|
process.exit(0);
|
|
}
|
|
const before = scenarioFiles.length;
|
|
scenarioFiles = scenarioFiles.filter((f) => {
|
|
const id = f.replace('.json', '');
|
|
return failedIds.has(id);
|
|
});
|
|
const excluded = before - scenarioFiles.length;
|
|
console.log(C.yellow(`실패 집중 모드: ${failedIds.size}개 실패 시나리오 대상`));
|
|
console.log(C.dim(` 소스: ${source}`));
|
|
console.log(C.dim(` 대상: ${Array.from(failedIds).join(', ')}`));
|
|
console.log(C.dim(` 건너뜀: ${excluded}개 성공 시나리오\n`));
|
|
}
|
|
|
|
const totalScenarios = scenarioFiles.length;
|
|
console.log(`시나리오: ${totalScenarios}개 발견\n`);
|
|
|
|
if (totalScenarios === 0) {
|
|
console.log(C.red('실행할 시나리오가 없습니다.'));
|
|
process.exit(1);
|
|
}
|
|
|
|
// Launch browser
|
|
const browser = await chromium.launch({
|
|
headless: HEADLESS,
|
|
args: [
|
|
`--window-position=1920,0`,
|
|
'--window-size=1920,1080',
|
|
'--disable-blink-features=AutomationControlled',
|
|
],
|
|
});
|
|
|
|
const context = await browser.newContext({
|
|
viewport: { width: 1920, height: 1080 },
|
|
locale: 'ko-KR',
|
|
ignoreHTTPSErrors: true,
|
|
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
|
});
|
|
|
|
const page = await context.newPage();
|
|
|
|
// Mask automation detection
|
|
await page.addInitScript(() => {
|
|
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
|
});
|
|
|
|
// Login
|
|
console.log(C.cyan('로그인 중...'));
|
|
try {
|
|
await page.goto(`${BASE_URL}/ko/login`, { waitUntil: 'domcontentloaded', timeout: 20000 });
|
|
await sleep(1000);
|
|
|
|
await page.fill('#userId', AUTH.username);
|
|
await page.fill('#password', AUTH.password);
|
|
await page.click("button[type='submit']");
|
|
await sleep(3000);
|
|
|
|
const url = page.url();
|
|
if (url.includes('/login')) {
|
|
console.log(C.red('로그인 실패! 대시보드로 이동하지 못함'));
|
|
await browser.close();
|
|
process.exit(1);
|
|
}
|
|
console.log(C.green('로그인 성공!\n'));
|
|
} catch (loginErr) {
|
|
console.log(C.red(`로그인 에러: ${loginErr.message}`));
|
|
await browser.close();
|
|
process.exit(1);
|
|
}
|
|
|
|
// ─── Helper: Run a list of scenario files and return results ────
|
|
async function runScenarioList(files, label, counter) {
|
|
const results = [];
|
|
for (let i = 0; i < files.length; i++) {
|
|
const file = files[i];
|
|
const scenarioPath = path.join(SCENARIOS_DIR, file);
|
|
counter.current++;
|
|
const num = `(${counter.current}/${counter.total})`;
|
|
|
|
process.stdout.write(`${C.dim(num)} ${label ? `[${label}] ` : ''}${file.replace('.json', '')} ... `);
|
|
|
|
let result;
|
|
const timeout = getScenarioTimeout(file);
|
|
try {
|
|
result = await Promise.race([
|
|
runScenario(page, scenarioPath),
|
|
sleep(timeout).then(() => ({
|
|
id: file.replace('.json', ''),
|
|
name: file.replace('.json', ''),
|
|
steps: [],
|
|
passed: 0,
|
|
failed: 0,
|
|
warned: 0,
|
|
totalSteps: 0,
|
|
apiSummary: null,
|
|
error: `Timeout (>${timeout / 1000}s)`,
|
|
stoppedReason: 'timeout',
|
|
currentUrl: '',
|
|
startTime: Date.now() - timeout,
|
|
endTime: Date.now(),
|
|
})),
|
|
]);
|
|
} catch (scenarioErr) {
|
|
result = {
|
|
id: file.replace('.json', ''),
|
|
name: file.replace('.json', ''),
|
|
steps: [],
|
|
passed: 0,
|
|
failed: 0,
|
|
warned: 0,
|
|
totalSteps: 0,
|
|
apiSummary: null,
|
|
error: scenarioErr.message,
|
|
stoppedReason: 'exception',
|
|
currentUrl: '',
|
|
startTime: Date.now(),
|
|
endTime: Date.now(),
|
|
};
|
|
}
|
|
|
|
results.push(result);
|
|
|
|
// Save report
|
|
const ts = getTimestamp();
|
|
saveReport(result, ts);
|
|
|
|
// Console output
|
|
const hasFail = result.failed > 0 || result.error;
|
|
const duration = ((result.endTime - result.startTime) / 1000).toFixed(1);
|
|
if (hasFail) {
|
|
console.log(`${C.red('FAIL')} ${C.dim(`(${result.passed}/${result.totalSteps} passed, ${duration}s)`)}`);
|
|
} else {
|
|
console.log(`${C.green('PASS')} ${C.dim(`(${result.passed}/${result.totalSteps}, ${duration}s)`)}`);
|
|
}
|
|
|
|
// After login scenario or if logged out, re-login
|
|
if (file === 'login.json' || page.url().includes('/login')) {
|
|
try {
|
|
await page.goto(`${BASE_URL}/ko/login`, { waitUntil: 'domcontentloaded', timeout: 10000 });
|
|
await sleep(500);
|
|
await page.fill('#userId', AUTH.username);
|
|
await page.fill('#password', AUTH.password);
|
|
await page.click("button[type='submit']");
|
|
await sleep(3000);
|
|
} catch (reloginErr) {
|
|
// Continue anyway
|
|
}
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
// ─── Stage Mode: Category-by-category execution ────────────
|
|
const STAGE_ORDER = ['accessibility', 'edge-case', 'performance', 'workflow', 'functional'];
|
|
const STAGE_NAMES = {
|
|
'accessibility': '접근성 검사',
|
|
'edge-case': '엣지 케이스',
|
|
'performance': '성능 테스트',
|
|
'workflow': '비즈니스 워크플로우',
|
|
'functional': '기능 테스트',
|
|
};
|
|
|
|
// Run scenarios
|
|
const allResults = [];
|
|
const iterationHistory = []; // Track iteration progress
|
|
const startTime = Date.now();
|
|
|
|
if (STAGE_MODE) {
|
|
// ─── STAGE MODE: Run by category ─────────────────────────
|
|
const categorized = {};
|
|
for (const file of scenarioFiles) {
|
|
const cat = getScenarioCategory(file);
|
|
if (!categorized[cat]) categorized[cat] = [];
|
|
categorized[cat].push(file);
|
|
}
|
|
|
|
let stageNum = 0;
|
|
const counter = { current: 0, total: totalScenarios };
|
|
|
|
for (const cat of STAGE_ORDER) {
|
|
const files = categorized[cat];
|
|
if (!files || files.length === 0) continue;
|
|
|
|
stageNum++;
|
|
const catName = STAGE_NAMES[cat] || cat;
|
|
console.log(C.bold(`\n── Stage ${stageNum}: ${catName} (${files.length}개) ──`));
|
|
|
|
const stageResults = await runScenarioList(files, catName, counter);
|
|
allResults.push(...stageResults);
|
|
|
|
// Stage summary
|
|
const stagePassed = stageResults.filter(r => !r.error && r.failed === 0).length;
|
|
const stageFailed = stageResults.length - stagePassed;
|
|
const stageRate = stageResults.length > 0 ? Math.round((stagePassed / stageResults.length) * 100) : 0;
|
|
|
|
console.log(C.cyan(`\n ▸ ${catName} 결과: ${stagePassed}/${stageResults.length} PASS (${stageRate}%)`));
|
|
if (stageFailed > 0) {
|
|
const failNames = stageResults.filter(r => r.failed > 0 || r.error).map(r => r.id);
|
|
console.log(C.red(` ▸ 실패: ${failNames.join(', ')}`));
|
|
}
|
|
}
|
|
|
|
} else {
|
|
// ─── NORMAL MODE: Run all sequentially ───────────────────
|
|
const counter = { current: 0, total: totalScenarios };
|
|
const results = await runScenarioList(scenarioFiles, '', counter);
|
|
allResults.push(...results);
|
|
}
|
|
|
|
// ─── Iteration 0 (initial run) summary ───────────────────────
|
|
const initialPassed = allResults.filter(r => !r.error && r.failed === 0).length;
|
|
const initialFailed = allResults.length - initialPassed;
|
|
|
|
iterationHistory.push({
|
|
iteration: 0,
|
|
label: '초기 실행',
|
|
total: allResults.length,
|
|
passed: initialPassed,
|
|
failed: initialFailed,
|
|
rate: allResults.length > 0 ? Math.round((initialPassed / allResults.length) * 100) : 0,
|
|
fixed: [],
|
|
stillFailing: allResults.filter(r => r.failed > 0 || r.error).map(r => r.id),
|
|
});
|
|
|
|
// ─── ITERATE MODE: Re-run failed scenarios ─────────────────
|
|
if (ITERATE && initialFailed > 0) {
|
|
for (let iteration = 1; iteration <= MAX_ITERATIONS; iteration++) {
|
|
const failedIds = allResults.filter(r => r.failed > 0 || r.error).map(r => r.id);
|
|
if (failedIds.length === 0) {
|
|
console.log(C.green(C.bold(`\n🎯 반복 ${iteration} 불필요: 모든 시나리오 PASS!`)));
|
|
break;
|
|
}
|
|
|
|
const failedFiles = failedIds.map(id => `${id}.json`).filter(f => fs.existsSync(path.join(SCENARIOS_DIR, f)));
|
|
if (failedFiles.length === 0) break;
|
|
|
|
console.log(C.bold(C.yellow(`\n═══ 반복 ${iteration}/${MAX_ITERATIONS}: ${failedFiles.length}개 실패 시나리오 재실행 ═══`)));
|
|
console.log(C.dim(` 대상: ${failedIds.join(', ')}\n`));
|
|
|
|
const counter = { current: 0, total: failedFiles.length };
|
|
const retryResults = await runScenarioList(failedFiles, `재시도 ${iteration}`, counter);
|
|
|
|
// Update allResults with retry results
|
|
const fixedInThisIteration = [];
|
|
for (const retryResult of retryResults) {
|
|
const idx = allResults.findIndex(r => r.id === retryResult.id);
|
|
if (idx >= 0) {
|
|
const wasFail = allResults[idx].failed > 0 || allResults[idx].error;
|
|
const nowPass = retryResult.failed === 0 && !retryResult.error;
|
|
if (wasFail && nowPass) {
|
|
fixedInThisIteration.push(retryResult.id);
|
|
}
|
|
allResults[idx] = retryResult; // Replace with latest result
|
|
}
|
|
}
|
|
|
|
// Iteration summary
|
|
const iterPassed = allResults.filter(r => !r.error && r.failed === 0).length;
|
|
const iterFailed = allResults.length - iterPassed;
|
|
const stillFailing = allResults.filter(r => r.failed > 0 || r.error).map(r => r.id);
|
|
|
|
iterationHistory.push({
|
|
iteration,
|
|
label: `반복 ${iteration}`,
|
|
total: allResults.length,
|
|
passed: iterPassed,
|
|
failed: iterFailed,
|
|
rate: allResults.length > 0 ? Math.round((iterPassed / allResults.length) * 100) : 0,
|
|
fixed: fixedInThisIteration,
|
|
stillFailing,
|
|
});
|
|
|
|
console.log(C.cyan(`\n ▸ 반복 ${iteration} 결과: ${iterPassed}/${allResults.length} PASS (${Math.round((iterPassed / allResults.length) * 100)}%)`));
|
|
if (fixedInThisIteration.length > 0) {
|
|
console.log(C.green(` ▸ 이번 반복에서 수정됨: ${fixedInThisIteration.join(', ')}`));
|
|
}
|
|
if (stillFailing.length > 0) {
|
|
console.log(C.red(` ▸ 여전히 실패: ${stillFailing.join(', ')}`));
|
|
} else {
|
|
console.log(C.green(C.bold(`\n🎯 모든 시나리오 PASS 달성! (반복 ${iteration}회 만에)`)));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
const totalTime = Date.now() - startTime;
|
|
|
|
// Generate summary report (with iteration history)
|
|
const summaryTs = getTimestamp();
|
|
let summaryMd = generateSummaryReport(allResults, totalTime, summaryTs);
|
|
|
|
// Append iteration history to summary if iterate mode was used
|
|
if (ITERATE && iterationHistory.length > 1) {
|
|
summaryMd += `\n## 반복 실행 이력\n`;
|
|
summaryMd += `| 반복 | 전체 | 성공 | 실패 | 성공률 | 수정됨 | 여전히 실패 |\n`;
|
|
summaryMd += `|:----:|:----:|:----:|:----:|:------:|--------|------------|\n`;
|
|
for (const h of iterationHistory) {
|
|
const fixedStr = h.fixed.length > 0 ? h.fixed.join(', ') : '-';
|
|
const failStr = h.stillFailing.length > 0 ? h.stillFailing.join(', ') : '-';
|
|
summaryMd += `| ${h.label} | ${h.total} | ${h.passed} | ${h.failed} | ${h.rate}% | ${fixedStr} | ${failStr} |\n`;
|
|
}
|
|
|
|
// Progress bar
|
|
summaryMd += `\n### 반복별 추이\n\`\`\`\n`;
|
|
for (const h of iterationHistory) {
|
|
const barLen = 40;
|
|
const filled = Math.round((h.rate / 100) * barLen);
|
|
const bar = '█'.repeat(filled) + '░'.repeat(barLen - filled);
|
|
summaryMd += `${h.label.padEnd(12)} ${bar} ${h.rate}% (${h.passed}/${h.total})\n`;
|
|
}
|
|
summaryMd += `\`\`\`\n`;
|
|
}
|
|
|
|
// Append stage summary if stage mode was used
|
|
if (STAGE_MODE) {
|
|
summaryMd += `\n## 단계별 실행 순서\n`;
|
|
summaryMd += `실행 순서: ${STAGE_ORDER.filter(c => allResults.some(r => getScenarioCategory(r.id) === c)).map(c => STAGE_NAMES[c] || c).join(' → ')}\n`;
|
|
}
|
|
|
|
const summaryPath = path.join(RESULTS_DIR, `E2E_FULL_TEST_SUMMARY_${summaryTs}.md`);
|
|
fs.writeFileSync(summaryPath, summaryMd, 'utf-8');
|
|
|
|
// Close browser
|
|
await browser.close();
|
|
|
|
// Print summary
|
|
const passCount = allResults.filter((r) => !r.error && r.failed === 0).length;
|
|
const failCount = allResults.length - passCount;
|
|
|
|
console.log(C.bold('\n=== 테스트 완료 ==='));
|
|
if (FAIL_ONLY) console.log(C.yellow(`[실패 집중 모드] 이전 실패 시나리오 ${totalScenarios}개만 실행`));
|
|
console.log(`전체: ${totalScenarios} | ${C.green(`성공: ${passCount}`)} | ${failCount > 0 ? C.red(`실패: ${failCount}`) : '실패: 0'}`);
|
|
if (ITERATE && iterationHistory.length > 1) {
|
|
const lastIter = iterationHistory[iterationHistory.length - 1];
|
|
const firstIter = iterationHistory[0];
|
|
const improvement = lastIter.passed - firstIter.passed;
|
|
if (improvement > 0) {
|
|
console.log(C.green(`반복 개선: +${improvement}개 PASS (${firstIter.rate}% → ${lastIter.rate}%)`));
|
|
}
|
|
console.log(`반복 횟수: ${iterationHistory.length - 1}회`);
|
|
}
|
|
console.log(`소요 시간: ${(totalTime / 1000 / 60).toFixed(1)}분`);
|
|
console.log(`요약 리포트: ${summaryPath}`);
|
|
console.log('');
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error(C.red(`\n치명적 에러: ${err.message}`));
|
|
console.error(err.stack);
|
|
process.exit(1);
|
|
});
|