2d973bc68a
step-executor.js: - wait_for_modal: interactive element 대기 옵션 추가 - wait_for_dialog_ready: 새 액션 (입력필드/버튼 렌더링 대기) - retryAction: progressive delay (500→1000→1500ms) + DOM context 진단 run-all.js: - --validate: 시나리오 JSON dry-run 검증 플래그 - verifyPageHealth(): 페이지 사전 건강성 체크 - diagnoseFail(): 실패 원인 자동 진단 - getPreviousRunResults(): 이전 실행 결과 파싱 - detectFlakyTests(): 3일간 flaky 테스트 감지 - generateSummaryReport(): 트렌드 분석/비교 기능 추가 전체 209/209 ALL PASS 검증 완료 (89.8분) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2254 lines
86 KiB
JavaScript
2254 lines
86 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* E2E 전체 테스트 독립 실행 러너
|
|
*
|
|
* MCP 오버헤드 없이 Node.js + Playwright 직접 사용
|
|
* 시나리오당 ~10초, 전체 ~15분 목표
|
|
*
|
|
* Usage:
|
|
* node e2e/runner/run-all.js # 전체 실행
|
|
* node e2e/runner/run-all.js --filter board # 파일명 필터
|
|
* node e2e/runner/run-all.js --workflow # 워크플로우만 실행
|
|
* node e2e/runner/run-all.js --headed # headed (기본값)
|
|
* node e2e/runner/run-all.js --headless # headless
|
|
* node e2e/runner/run-all.js --exclude sales # 파일명에 "sales" 포함된 것 제외
|
|
* node e2e/runner/run-all.js --skip-passed # 이미 성공한 시나리오 건너뛰기
|
|
* node e2e/runner/run-all.js --iterate # 실패 시나리오 자동 재실행 (최대 3회)
|
|
* node e2e/runner/run-all.js --iterate 5 # 실패 시나리오 자동 재실행 (최대 5회)
|
|
* node e2e/runner/run-all.js --stage # 카테고리별 단계 실행 (accessibility → edge → perf → workflow → functional)
|
|
* node e2e/runner/run-all.js --fail-only # 최근 요약 리포트의 실패 시나리오만 실행
|
|
* node e2e/runner/run-all.js --fail-only --iterate # 실패 시나리오만 반복 실행
|
|
* node e2e/runner/run-all.js --group accounting # 모듈 그룹 실행 (회계관리)
|
|
* node e2e/runner/run-all.js --group hr,approval # 여러 그룹 조합 (쉼표 구분)
|
|
* node e2e/runner/run-all.js --group search # 테스트 유형 그룹 (검색 테스트)
|
|
* node e2e/runner/run-all.js --groups # 사용 가능한 그룹 목록 출력
|
|
*/
|
|
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
// ─── Resolve Playwright from react/node_modules ─────────────
|
|
const SAM_ROOT = path.resolve(__dirname, '..', '..');
|
|
const PW_PATH = path.join(SAM_ROOT, 'react', 'node_modules', 'playwright');
|
|
const { chromium } = require(PW_PATH);
|
|
|
|
// ─── Config ─────────────────────────────────────────────────
|
|
const BASE_URL = 'https://dev.codebridge-x.com';
|
|
const AUTH = { username: 'TestUser5', password: 'password123!' };
|
|
const SCENARIOS_DIR = path.join(SAM_ROOT, 'e2e', 'scenarios');
|
|
const RESULTS_DIR = path.join(SAM_ROOT, 'e2e', 'results', 'hotfix');
|
|
const SUCCESS_DIR = path.join(RESULTS_DIR, 'success');
|
|
const SCREENSHOTS_DIR = path.join(RESULTS_DIR, 'screenshots');
|
|
const EXECUTOR_PATH = path.join(SAM_ROOT, 'e2e', 'runner', 'step-executor.js');
|
|
const DASHBOARD_URL = `${BASE_URL}/dashboard`;
|
|
|
|
// CLI args
|
|
const args = process.argv.slice(2);
|
|
const HEADLESS = args.includes('--headless');
|
|
const WORKFLOW_ONLY = args.includes('--workflow');
|
|
const FILTER = (() => {
|
|
const idx = args.indexOf('--filter');
|
|
return idx >= 0 && args[idx + 1] ? args[idx + 1] : null;
|
|
})();
|
|
const EXCLUDE = (() => {
|
|
const idx = args.indexOf('--exclude');
|
|
return idx >= 0 && args[idx + 1] ? args[idx + 1] : null;
|
|
})();
|
|
const SKIP_PASSED = args.includes('--skip-passed');
|
|
const ITERATE = args.includes('--iterate');
|
|
const MAX_ITERATIONS = (() => {
|
|
const idx = args.indexOf('--iterate');
|
|
if (idx >= 0 && args[idx + 1] && !args[idx + 1].startsWith('--')) {
|
|
const n = parseInt(args[idx + 1], 10);
|
|
return isNaN(n) ? 3 : Math.min(Math.max(n, 1), 10);
|
|
}
|
|
return 3;
|
|
})();
|
|
const STAGE_MODE = args.includes('--stage');
|
|
const FAIL_ONLY = args.includes('--fail-only');
|
|
const GROUP = (() => {
|
|
const idx = args.indexOf('--group');
|
|
return idx >= 0 && args[idx + 1] ? args[idx + 1] : null;
|
|
})();
|
|
const LIST_GROUPS = args.includes('--groups');
|
|
const VALIDATE_ONLY = args.includes('--validate');
|
|
|
|
// ─── Group Definitions ──────────────────────────────────────
|
|
|
|
// 테스트 유형 그룹 (파일명 prefix 매칭)
|
|
const TYPE_GROUPS = {
|
|
search: ['search-'],
|
|
a11y: ['a11y-'],
|
|
perf: ['perf-'],
|
|
edge: ['edge-'],
|
|
crud: ['full-crud-', 'create-delete-'],
|
|
workflow: ['workflow-'],
|
|
input: ['input-fields-'],
|
|
batch: ['batch-'],
|
|
reload: ['reload-persist-'],
|
|
detail: ['detail-'],
|
|
pagination: ['pagination-sort-'],
|
|
api: ['api-health-'],
|
|
validation: ['form-validation-'],
|
|
};
|
|
|
|
// 모듈 그룹 (menuNavigation.level1 매칭)
|
|
const MODULE_GROUPS = {
|
|
accounting: ['회계관리'],
|
|
hr: ['인사관리'],
|
|
sales: ['판매관리'],
|
|
board: ['게시판'],
|
|
settings: ['설정'],
|
|
production: ['생산관리'],
|
|
material: ['자재관리'],
|
|
approval: ['결재관리'],
|
|
quality: ['품질관리'],
|
|
customer: ['고객센터'],
|
|
purchase: ['구매관리'],
|
|
standard: ['기준정보 관리'],
|
|
shipping: ['출고관리'],
|
|
item: ['품목관리'],
|
|
dashboard: ['시스템 대시보드'],
|
|
};
|
|
|
|
/**
|
|
* --group 필터링: 모듈 그룹(level1) 또는 테스트 유형 그룹(파일명 prefix) 매칭
|
|
*/
|
|
function filterByGroup(files, groupStr) {
|
|
const groupNames = groupStr.split(',').map(g => g.trim().toLowerCase());
|
|
// 잘못된 그룹명 경고
|
|
const invalidGroups = groupNames.filter(g => !TYPE_GROUPS[g] && !MODULE_GROUPS[g]);
|
|
if (invalidGroups.length > 0) {
|
|
const allGroupNames = [...Object.keys(TYPE_GROUPS), ...Object.keys(MODULE_GROUPS)].sort();
|
|
console.log(C.red(`알 수 없는 그룹: ${invalidGroups.join(', ')}`));
|
|
console.log(C.dim(`사용 가능한 그룹: ${allGroupNames.join(', ')}`));
|
|
console.log(C.dim(`전체 목록: node e2e/runner/run-all.js --groups\n`));
|
|
}
|
|
// 모듈 그룹이 포함된 경우 level1 캐시를 한 번에 구축
|
|
const needsLevel1 = groupNames.some(g => MODULE_GROUPS[g]);
|
|
const level1Cache = {};
|
|
if (needsLevel1) {
|
|
for (const f of files) {
|
|
try {
|
|
const data = JSON.parse(fs.readFileSync(path.join(SCENARIOS_DIR, f), 'utf-8'));
|
|
level1Cache[f] = data.menuNavigation?.level1 || null;
|
|
} catch { level1Cache[f] = null; }
|
|
}
|
|
}
|
|
return files.filter(f => {
|
|
const basename = f.replace('.json', '');
|
|
return groupNames.some(groupName => {
|
|
// 테스트 유형 그룹 (prefix 매칭)
|
|
if (TYPE_GROUPS[groupName]) {
|
|
return TYPE_GROUPS[groupName].some(prefix => basename.startsWith(prefix));
|
|
}
|
|
// 모듈 그룹 (level1 매칭)
|
|
if (MODULE_GROUPS[groupName]) {
|
|
return MODULE_GROUPS[groupName].includes(level1Cache[f]);
|
|
}
|
|
return false;
|
|
});
|
|
});
|
|
}
|
|
|
|
// ─── Helpers ────────────────────────────────────────────────
|
|
|
|
function getTimestamp() {
|
|
const n = new Date();
|
|
const pad = (v) => v.toString().padStart(2, '0');
|
|
return `${n.getFullYear()}-${pad(n.getMonth() + 1)}-${pad(n.getDate())}_${pad(n.getHours())}-${pad(n.getMinutes())}-${pad(n.getSeconds())}`;
|
|
}
|
|
|
|
function ensureDir(dir) {
|
|
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
}
|
|
|
|
function sleep(ms) {
|
|
return new Promise((r) => setTimeout(r, ms));
|
|
}
|
|
|
|
/**
|
|
* success/ 폴더의 OK- 리포트에서 이미 성공한 시나리오 ID 집합을 반환.
|
|
* 파일명 형식: OK-{scenario-id}_{timestamp}.md
|
|
*/
|
|
function getPassedScenarioIds() {
|
|
const passed = new Set();
|
|
if (!fs.existsSync(SUCCESS_DIR)) return passed;
|
|
const files = fs.readdirSync(SUCCESS_DIR).filter(f => f.startsWith('OK-') && f.endsWith('.md'));
|
|
for (const f of files) {
|
|
// OK-accounting-deposit_2026-02-23_20-22-27.md → accounting-deposit
|
|
const withoutPrefix = f.substring(3); // remove "OK-"
|
|
const tsMatch = withoutPrefix.match(/_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\.md$/);
|
|
if (tsMatch) {
|
|
const id = withoutPrefix.substring(0, tsMatch.index);
|
|
passed.add(id);
|
|
}
|
|
}
|
|
return passed;
|
|
}
|
|
|
|
/**
|
|
* 최근 E2E_FULL_TEST_SUMMARY 리포트에서 실패한 시나리오 ID 집합을 반환.
|
|
* --fail-only 모드에서 사용.
|
|
*
|
|
* 파싱 전략:
|
|
* 1차: "### ❌ {name} ({scenario-id})" 형식에서 ID 추출 (가장 신뢰)
|
|
* 2차: Fail-{scenario-id}_{timestamp}.md 파일명에서 추출 (보완)
|
|
*/
|
|
function getFailedScenarioIds() {
|
|
const failed = new Set();
|
|
if (!fs.existsSync(RESULTS_DIR)) return { ids: failed, source: null };
|
|
|
|
// E2E_FULL_TEST_SUMMARY_*.md 파일 중 가장 최근 것 찾기
|
|
const summaryFiles = fs.readdirSync(RESULTS_DIR)
|
|
.filter(f => f.startsWith('E2E_FULL_TEST_SUMMARY_') && f.endsWith('.md'))
|
|
.sort()
|
|
.reverse();
|
|
|
|
if (summaryFiles.length === 0) {
|
|
console.log(' (이전 요약 리포트 없음 - Fail- 파일에서 추출)');
|
|
// Fail- 리포트에서 추출
|
|
const failFiles = fs.readdirSync(RESULTS_DIR)
|
|
.filter(f => f.startsWith('Fail-') && f.endsWith('.md'));
|
|
for (const f of failFiles) {
|
|
const withoutPrefix = f.substring(5); // remove "Fail-"
|
|
const tsMatch = withoutPrefix.match(/_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\.md$/);
|
|
if (tsMatch) {
|
|
failed.add(withoutPrefix.substring(0, tsMatch.index));
|
|
}
|
|
}
|
|
return { ids: failed, source: 'Fail-*.md files' };
|
|
}
|
|
|
|
const latestSummary = summaryFiles[0];
|
|
const content = fs.readFileSync(path.join(RESULTS_DIR, latestSummary), 'utf-8');
|
|
|
|
// "### ❌ 시나리오명 (scenario-id)" 패턴에서 ID 추출
|
|
const lines = content.split('\n');
|
|
for (const line of lines) {
|
|
const match = line.match(/^###\s*❌\s*.+\(([a-zA-Z0-9_-]+)\)\s*$/);
|
|
if (match) {
|
|
failed.add(match[1]);
|
|
}
|
|
}
|
|
|
|
// 보완: Fail- 리포트에서도 추출 (요약 파싱 실패 시)
|
|
if (failed.size === 0) {
|
|
const failFiles = fs.readdirSync(RESULTS_DIR)
|
|
.filter(f => f.startsWith('Fail-') && f.endsWith('.md'));
|
|
for (const f of failFiles) {
|
|
const withoutPrefix = f.substring(5);
|
|
const tsMatch = withoutPrefix.match(/_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\.md$/);
|
|
if (tsMatch) {
|
|
failed.add(withoutPrefix.substring(0, tsMatch.index));
|
|
}
|
|
}
|
|
}
|
|
|
|
return { ids: failed, source: latestSummary };
|
|
}
|
|
|
|
/** Color console helpers */
|
|
const C = {
|
|
green: (s) => `\x1b[32m${s}\x1b[0m`,
|
|
red: (s) => `\x1b[31m${s}\x1b[0m`,
|
|
yellow: (s) => `\x1b[33m${s}\x1b[0m`,
|
|
cyan: (s) => `\x1b[36m${s}\x1b[0m`,
|
|
dim: (s) => `\x1b[2m${s}\x1b[0m`,
|
|
bold: (s) => `\x1b[1m${s}\x1b[0m`,
|
|
};
|
|
|
|
/**
|
|
* --groups: 사용 가능한 그룹 목록 + 시나리오 수 출력
|
|
*/
|
|
function listGroups() {
|
|
const allFiles = fs.readdirSync(SCENARIOS_DIR)
|
|
.filter(f => f.endsWith('.json') && !f.startsWith('_'))
|
|
.sort();
|
|
|
|
// level1 캐시 구축
|
|
const level1Cache = {};
|
|
for (const f of allFiles) {
|
|
try {
|
|
const data = JSON.parse(fs.readFileSync(path.join(SCENARIOS_DIR, f), 'utf-8'));
|
|
level1Cache[f] = data.menuNavigation?.level1 || null;
|
|
} catch { level1Cache[f] = null; }
|
|
}
|
|
|
|
console.log(C.bold('\n=== 사용 가능한 그룹 목록 ===\n'));
|
|
|
|
console.log(C.cyan(C.bold('모듈 그룹 (menuNavigation.level1 기반):')));
|
|
const moduleEntries = Object.entries(MODULE_GROUPS).sort((a, b) => {
|
|
const countA = allFiles.filter(f => a[1].includes(level1Cache[f])).length;
|
|
const countB = allFiles.filter(f => b[1].includes(level1Cache[f])).length;
|
|
return countB - countA;
|
|
});
|
|
for (const [name, level1s] of moduleEntries) {
|
|
const count = allFiles.filter(f => level1s.includes(level1Cache[f])).length;
|
|
if (count > 0) {
|
|
console.log(` ${C.green(name.padEnd(14))} ${String(count).padStart(3)}개 (${level1s.join(', ')})`);
|
|
}
|
|
}
|
|
|
|
console.log('');
|
|
console.log(C.cyan(C.bold('테스트 유형 그룹 (파일명 prefix 기반):')));
|
|
const typeEntries = Object.entries(TYPE_GROUPS).sort((a, b) => {
|
|
const countA = allFiles.filter(f => a[1].some(p => f.replace('.json', '').startsWith(p))).length;
|
|
const countB = allFiles.filter(f => b[1].some(p => f.replace('.json', '').startsWith(p))).length;
|
|
return countB - countA;
|
|
});
|
|
for (const [name, prefixes] of typeEntries) {
|
|
const count = allFiles.filter(f => prefixes.some(p => f.replace('.json', '').startsWith(p))).length;
|
|
if (count > 0) {
|
|
console.log(` ${C.green(name.padEnd(14))} ${String(count).padStart(3)}개 (${prefixes.join(', ')})`);
|
|
}
|
|
}
|
|
|
|
console.log(`\n${C.dim('사용법: node e2e/runner/run-all.js --group <그룹명>[,<그룹명>...]')}`);
|
|
console.log(`${C.dim(' 예: --group accounting --group hr,approval --group search')}\n`);
|
|
}
|
|
|
|
// ─── Scenario Validation ────────────────────────────────────
|
|
|
|
/**
|
|
* --validate: Dry-run scenario validation without browser.
|
|
* Checks JSON structure, required fields, deprecated patterns, and action validity.
|
|
*/
|
|
function validateScenarios(files) {
|
|
const VALID_ACTIONS = new Set([
|
|
'click', 'click_nth', 'click_row', 'click_first_row', 'click_button',
|
|
'click_dialog_confirm', 'click_and_confirm', 'click_if_exists',
|
|
'fill', 'fill_nth', 'fill_form', 'fill_and_wait', 'clear', 'edit_field',
|
|
'select', 'select_dropdown', 'select_filter',
|
|
'check', 'uncheck', 'check_nth', 'uncheck_nth',
|
|
'wait', 'wait_for_element', 'wait_for_table', 'wait_for_modal', 'wait_for_navigation',
|
|
'verify_element', 'verify_checks', 'verify_text', 'verify_url', 'verify_url_stability',
|
|
'verify_table', 'verify_table_structure', 'verify_data', 'verify_detail',
|
|
'verify_not_mockup', 'verify_dialog', 'verify_input_value', 'verify_page',
|
|
'verify_console', 'verify_data_change', 'verify_edit_mode', 'verify_toast',
|
|
'save_url', 'extract_from_url', 'capture', 'close_modal', 'close_modal_if_open',
|
|
'scrollAndFind', 'evaluate', 'search', 'press_key', 'blur', 'hover',
|
|
'generate_timestamp', 'navigate', 'reload', 'navigate_back', 'menu_navigate',
|
|
'workflow_context_save', 'workflow_context_load', 'cross_module_verify',
|
|
'measure_performance', 'measure_api_performance', 'assert_performance',
|
|
'fill_boundary', 'rapid_click', 'accessibility_audit', 'keyboard_navigate',
|
|
]);
|
|
|
|
// Common aliases that should also be accepted
|
|
const VALID_ALIASES = new Set([
|
|
// Fill aliases
|
|
'type', 'input', 'text', 'textarea', 'email', 'password', 'number',
|
|
'clear_and_type', 'clear-and-type', 'change_date', 'change_date_range',
|
|
'date_range', 'date', 'datepicker', 'setDateRange', 'timepicker', 'login',
|
|
// Click aliases
|
|
'click+confirm', 'click_download', 'click_dropdown', 'click_checkbox',
|
|
'clickFirstRow', 'clickInModal', 'delete', 'download',
|
|
// Navigation aliases
|
|
'navigateBack', 'goBack', 'navigation', 'directNavigation',
|
|
'navigateViaMenuClick', 'refresh',
|
|
// Wait aliases
|
|
'waitForModal', 'waitForNavigation', 'waitForTable',
|
|
'waitForDialogReady', 'wait_dialog_ready', 'wait_for_dialog_ready',
|
|
// Verify aliases
|
|
'verify', 'verify_elements', 'verify_table_data', 'verify_table_structure',
|
|
'verify_field', 'verify_checkbox', 'verify_action_buttons', 'verify_pagination',
|
|
'verify_summary', 'verify_totals', 'verify_search_result', 'verify_row_count',
|
|
'verify_vendor_info', 'verify_detail_info', 'verify_data_update',
|
|
'verify_transactions_update', 'verify_transaction_table', 'verify_calculated_value',
|
|
'verifyButtonExists', 'verifyUrl', 'verifyNoErrorPage',
|
|
// Select aliases
|
|
'combobox', 'select_option', 'select_or_click',
|
|
// Modal aliases
|
|
'closeModal', 'modalClose', 'openModal', 'checkModalOpen',
|
|
'fillInModal', 'selectInModal',
|
|
// Other aliases
|
|
'confirm_dialog', 'store', 'getCurrentUrl', 'clearSearch',
|
|
'keypress', 'press', 'pressKey', 'generateTimestamp', 'random',
|
|
'setupDownloadListener', 'saveDownloadedFile', 'verifyDownload',
|
|
'verifyDownloadedFile', 'screenshot', 'capture',
|
|
'toggle_switch', 'radio', 'resize', 'log', 'manualVerification',
|
|
'composite', 'hierarchy', 'permission', 'ifStillFailed', 'tryAlternativeUrls',
|
|
'element', 'elementExists', 'tableExists', 'tabsExist',
|
|
'error_message', 'warning', 'url', 'URL_STABILITY', 'checkFor404',
|
|
'expectResponse', 'assertResponse', 'apiResponse',
|
|
'findRow', 'scroll',
|
|
// Extended aliases
|
|
'a11y_audit', 'accessibility', 'perf_measure', 'performance',
|
|
'boundary_fill', 'rapid', 'save_context', 'context_save',
|
|
'load_context', 'context_load', 'noop',
|
|
]);
|
|
|
|
const DEPRECATED_PATTERNS = [
|
|
{ pattern: 'window.__API_LOGS__', fix: 'window.__E2E__.getApiLogs().logs', severity: 'error' },
|
|
{ pattern: 'window.__API_ERRORS__', fix: 'window.__E2E__.getApiLogs().errors', severity: 'error' },
|
|
];
|
|
|
|
let totalIssues = 0;
|
|
let errorCount = 0;
|
|
let warnCount = 0;
|
|
|
|
console.log(C.bold('\n=== 시나리오 정합성 검증 ===\n'));
|
|
|
|
for (const f of files) {
|
|
const fPath = path.join(SCENARIOS_DIR, f);
|
|
const issues = [];
|
|
|
|
try {
|
|
const raw = fs.readFileSync(fPath, 'utf-8');
|
|
let data;
|
|
try {
|
|
data = JSON.parse(raw);
|
|
} catch (parseErr) {
|
|
issues.push({ severity: 'error', msg: `JSON 파싱 실패: ${parseErr.message}` });
|
|
errorCount++;
|
|
totalIssues++;
|
|
console.log(`${C.red('✘')} ${f}`);
|
|
issues.forEach(i => console.log(` ${i.severity === 'error' ? C.red('ERROR') : C.yellow('WARN')}: ${i.msg}`));
|
|
continue;
|
|
}
|
|
|
|
// Required fields
|
|
if (!data.id) issues.push({ severity: 'warn', msg: 'id 필드 누락' });
|
|
if (!data.name) issues.push({ severity: 'warn', msg: 'name 필드 누락' });
|
|
if (!data.steps || !Array.isArray(data.steps)) {
|
|
issues.push({ severity: 'error', msg: 'steps 배열 누락 또는 잘못된 형식' });
|
|
}
|
|
if (!data.menuNavigation) issues.push({ severity: 'warn', msg: 'menuNavigation 누락' });
|
|
|
|
// Step validation
|
|
if (data.steps && Array.isArray(data.steps)) {
|
|
for (const step of data.steps) {
|
|
const action = step.action || step.type;
|
|
if (action && !VALID_ACTIONS.has(action) && !VALID_ALIASES.has(action)) {
|
|
issues.push({ severity: 'warn', msg: `스텝 #${step.id}: 알 수 없는 액션 "${action}"` });
|
|
}
|
|
// Duplicate step IDs
|
|
const idCount = data.steps.filter(s => s.id === step.id).length;
|
|
if (idCount > 1 && step === data.steps.find(s => s.id === step.id)) {
|
|
issues.push({ severity: 'error', msg: `스텝 ID ${step.id} 중복 (${idCount}회)` });
|
|
}
|
|
}
|
|
|
|
// Step ID sequencing
|
|
const ids = data.steps.map(s => s.id).filter(id => typeof id === 'number');
|
|
for (let i = 1; i < ids.length; i++) {
|
|
if (ids[i] <= ids[i - 1]) {
|
|
issues.push({ severity: 'warn', msg: `스텝 ID 순서 비정상: #${ids[i-1]} → #${ids[i]}` });
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Deprecated pattern check (in raw text)
|
|
for (const dp of DEPRECATED_PATTERNS) {
|
|
if (raw.includes(dp.pattern)) {
|
|
issues.push({ severity: dp.severity, msg: `deprecated 패턴: "${dp.pattern}" → "${dp.fix}"` });
|
|
}
|
|
}
|
|
|
|
// Enabled check
|
|
if (data.enabled === false) {
|
|
issues.push({ severity: 'warn', msg: 'enabled: false (비활성 시나리오)' });
|
|
}
|
|
|
|
// menuNavigation consistency
|
|
if (data.menuNavigation && data.steps) {
|
|
const menuStep = data.steps.find(s => s.action === 'menu_navigate');
|
|
if (menuStep) {
|
|
if (menuStep.level1 !== data.menuNavigation.level1 || menuStep.level2 !== data.menuNavigation.level2) {
|
|
issues.push({ severity: 'warn', msg: `menuNavigation과 menu_navigate 스텝 불일치: ${data.menuNavigation.level1}>${data.menuNavigation.level2} vs ${menuStep.level1}>${menuStep.level2}` });
|
|
}
|
|
}
|
|
}
|
|
|
|
} catch (readErr) {
|
|
issues.push({ severity: 'error', msg: `파일 읽기 실패: ${readErr.message}` });
|
|
}
|
|
|
|
// Tally
|
|
const errors = issues.filter(i => i.severity === 'error');
|
|
const warns = issues.filter(i => i.severity === 'warn');
|
|
errorCount += errors.length;
|
|
warnCount += warns.length;
|
|
totalIssues += issues.length;
|
|
|
|
if (issues.length > 0) {
|
|
const icon = errors.length > 0 ? C.red('✘') : C.yellow('△');
|
|
console.log(`${icon} ${f} (${errors.length} errors, ${warns.length} warns)`);
|
|
issues.forEach(i => {
|
|
const prefix = i.severity === 'error' ? C.red(' ERROR') : C.yellow(' WARN');
|
|
console.log(`${prefix}: ${i.msg}`);
|
|
});
|
|
}
|
|
}
|
|
|
|
// Summary
|
|
const okCount = files.length - (new Set([...files.filter(f => {
|
|
try {
|
|
const raw = fs.readFileSync(path.join(SCENARIOS_DIR, f), 'utf-8');
|
|
return DEPRECATED_PATTERNS.some(dp => raw.includes(dp.pattern));
|
|
} catch { return false; }
|
|
})])).size;
|
|
|
|
console.log(`\n${C.bold('─── 검증 결과 ───')}`);
|
|
console.log(` 전체 시나리오: ${C.bold(files.length)}개`);
|
|
console.log(` 이슈: ${totalIssues > 0 ? C.yellow(totalIssues) : C.green('0')}건 (에러: ${errorCount}, 경고: ${warnCount})`);
|
|
console.log(` ${totalIssues === 0 ? C.green('✅ 모든 시나리오 정합성 통과') : C.yellow(`⚠️ ${totalIssues}건의 이슈 발견`)}\n`);
|
|
|
|
return { total: files.length, errors: errorCount, warns: warnCount };
|
|
}
|
|
|
|
// ─── Executor Injection ─────────────────────────────────────
|
|
|
|
const executorCode = fs.readFileSync(EXECUTOR_PATH, 'utf-8');
|
|
|
|
async function injectExecutor(page) {
|
|
await page.evaluate(executorCode);
|
|
const initResult = await page.evaluate(() => JSON.stringify(window.__E2E__.init()));
|
|
const parsed = JSON.parse(initResult);
|
|
if (!parsed.ready) throw new Error('step-executor init failed');
|
|
return parsed;
|
|
}
|
|
|
|
// ─── Menu Navigation ────────────────────────────────────────
|
|
|
|
/**
|
|
* Wait for sidebar to render with clickable menu items.
|
|
* Returns true if sidebar is ready, false otherwise.
|
|
*/
|
|
async function waitForSidebarReady(page, timeout = 8000) {
|
|
try {
|
|
await page.waitForFunction(
|
|
() => {
|
|
const sidebar = document.querySelector('.sidebar-scroll, [data-sidebar="content"]');
|
|
if (!sidebar) return false;
|
|
const items = sidebar.querySelectorAll('a, button, [role="button"], [role="menuitem"]');
|
|
return Array.from(items).filter(el => {
|
|
const t = (el.innerText || '').trim();
|
|
return t.length > 1 && t.length < 30;
|
|
}).length >= 3;
|
|
},
|
|
null,
|
|
{ timeout }
|
|
);
|
|
return true;
|
|
} catch (e) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Ensure sidebar is expanded (not icon-only mode).
|
|
* Checks localStorage and reloads from Node.js side if needed.
|
|
*/
|
|
async function ensureSidebarExpanded(page) {
|
|
const needsReload = await page.evaluate(() => {
|
|
try {
|
|
const raw = localStorage.getItem('sam-menu');
|
|
if (raw) {
|
|
const data = JSON.parse(raw);
|
|
if (data.state && data.state.sidebarCollapsed) {
|
|
data.state.sidebarCollapsed = false;
|
|
localStorage.setItem('sam-menu', JSON.stringify(data));
|
|
return true; // needs reload to apply
|
|
}
|
|
}
|
|
} catch (e) {}
|
|
return false;
|
|
});
|
|
|
|
if (needsReload) {
|
|
await page.reload({ waitUntil: 'load', timeout: 12000 });
|
|
await sleep(1500);
|
|
}
|
|
}
|
|
|
|
async function navigateViaMenu(page, level1, level2) {
|
|
// Phase 0: Ensure sidebar is rendered and expanded
|
|
let sidebarReady = await waitForSidebarReady(page, 6000);
|
|
if (!sidebarReady) {
|
|
console.log(C.yellow(` [NAV] sidebar not ready, reloading...`));
|
|
try {
|
|
await page.reload({ waitUntil: 'load', timeout: 12000 });
|
|
await sleep(2000);
|
|
} catch (e) { /* ignore */ }
|
|
sidebarReady = await waitForSidebarReady(page, 8000);
|
|
if (!sidebarReady) {
|
|
console.log(C.red(` [NAV] sidebar still not rendered after reload! URL: ${page.url()}`));
|
|
return false;
|
|
}
|
|
}
|
|
|
|
await ensureSidebarExpanded(page);
|
|
|
|
// Phase 1: Collapse all open accordions, scroll to top
|
|
await page.evaluate(() => {
|
|
const collapseBtn = Array.from(document.querySelectorAll('button, [role="button"]'))
|
|
.find(el => el.innerText?.trim() === '모두 접기');
|
|
if (collapseBtn) collapseBtn.click();
|
|
});
|
|
await sleep(400);
|
|
|
|
await page.evaluate(() => {
|
|
const sidebar = document.querySelector('.sidebar-scroll, [data-sidebar="content"], nav');
|
|
if (sidebar) sidebar.scrollTo({ top: 0, behavior: 'instant' });
|
|
});
|
|
await sleep(300);
|
|
|
|
// Phase 2: Find and click L1 menu (accordion header)
|
|
// Use innerText for accurate visible-text matching (textContent includes hidden child text)
|
|
const l1Found = await page.evaluate(
|
|
async ({ l1Text }) => {
|
|
const sidebar = document.querySelector('.sidebar-scroll, [data-sidebar="content"], nav');
|
|
const maxScrollAttempts = 25;
|
|
|
|
for (let i = 0; i < maxScrollAttempts; i++) {
|
|
// Collect only direct menu buttons/links (not their children)
|
|
const candidates = Array.from(
|
|
document.querySelectorAll(
|
|
'[data-sidebar="content"] > * > * > button, ' +
|
|
'[data-sidebar="content"] > * > * > a, ' +
|
|
'.sidebar-scroll button, .sidebar-scroll a, ' +
|
|
'nav button, nav a, ' +
|
|
'[role="menuitem"], [role="treeitem"]'
|
|
)
|
|
);
|
|
|
|
for (const el of candidates) {
|
|
// Use innerText for accurate match (excludes hidden sub-menus)
|
|
const elText = (el.innerText || '').trim();
|
|
// Also check only the first line (in case submenu text is appended)
|
|
const firstLine = elText.split('\n')[0].trim();
|
|
|
|
if (firstLine === l1Text || elText === l1Text) {
|
|
el.scrollIntoView({ behavior: 'instant', block: 'center' });
|
|
await new Promise(r => setTimeout(r, 150));
|
|
el.click();
|
|
return { found: true, text: firstLine, attempt: i };
|
|
}
|
|
}
|
|
|
|
// Fallback: startsWith match (e.g., "회계관리 14" badge suffix)
|
|
for (const el of candidates) {
|
|
const firstLine = (el.innerText || '').split('\n')[0].trim();
|
|
if (firstLine.startsWith(l1Text) && firstLine.length < l1Text.length + 10) {
|
|
el.scrollIntoView({ behavior: 'instant', block: 'center' });
|
|
await new Promise(r => setTimeout(r, 150));
|
|
el.click();
|
|
return { found: true, text: firstLine, attempt: i, partial: true };
|
|
}
|
|
}
|
|
|
|
if (sidebar) {
|
|
sidebar.scrollBy({ top: 120, behavior: 'instant' });
|
|
await new Promise(r => setTimeout(r, 120));
|
|
}
|
|
}
|
|
return { found: false };
|
|
},
|
|
{ l1Text: level1 }
|
|
);
|
|
|
|
if (!l1Found.found) {
|
|
// Collect debug info before returning
|
|
const debugTexts = await page.evaluate(() => {
|
|
const items = document.querySelectorAll('nav a, nav button, [role="menuitem"], [role="treeitem"]');
|
|
return Array.from(items)
|
|
.map(el => (el.innerText || '').split('\n')[0].trim())
|
|
.filter(t => t.length > 1 && t.length < 25)
|
|
.filter((t, i, arr) => arr.indexOf(t) === i)
|
|
.slice(0, 20);
|
|
});
|
|
console.log(C.red(` [NAV] L1 "${level1}" not found.`));
|
|
console.log(C.dim(` [NAV] Available: ${debugTexts.join(', ')}`));
|
|
return false;
|
|
}
|
|
await sleep(800); // Wait for accordion animation to expand
|
|
|
|
// Phase 3: Verify accordion expanded (L2 items should now be visible)
|
|
if (level2) {
|
|
// Wait for L2 items to appear after accordion expansion
|
|
let l2Visible = false;
|
|
for (let retryWait = 0; retryWait < 3; retryWait++) {
|
|
l2Visible = await page.evaluate(
|
|
(l2Text) => {
|
|
const items = document.querySelectorAll('a, button, [role="menuitem"], [role="treeitem"]');
|
|
return Array.from(items).some(el => {
|
|
const t = (el.innerText || '').trim();
|
|
return t === l2Text || t.includes(l2Text);
|
|
});
|
|
},
|
|
level2
|
|
);
|
|
if (l2Visible) break;
|
|
// Accordion might not have expanded - try clicking L1 again
|
|
if (retryWait === 1) {
|
|
await page.evaluate(
|
|
async ({ l1Text }) => {
|
|
const candidates = document.querySelectorAll('nav button, nav a, [role="menuitem"], [role="treeitem"]');
|
|
for (const el of candidates) {
|
|
const firstLine = (el.innerText || '').split('\n')[0].trim();
|
|
if (firstLine === l1Text || firstLine.startsWith(l1Text)) {
|
|
el.click();
|
|
break;
|
|
}
|
|
}
|
|
},
|
|
{ l1Text: level1 }
|
|
);
|
|
}
|
|
await sleep(600);
|
|
}
|
|
|
|
// Phase 4: Find and click L2 menu item
|
|
const l2Found = await page.evaluate(
|
|
async ({ l2Text }) => {
|
|
const sidebar = document.querySelector('.sidebar-scroll, [data-sidebar="content"], nav');
|
|
const maxAttempts = 15;
|
|
|
|
for (let i = 0; i < maxAttempts; i++) {
|
|
const items = Array.from(
|
|
document.querySelectorAll('a, button, [role="menuitem"], [role="treeitem"]')
|
|
);
|
|
|
|
// Exact match on innerText (most reliable)
|
|
let match = items.find(el => (el.innerText || '').trim() === l2Text);
|
|
|
|
// Partial match fallback
|
|
if (!match) {
|
|
match = items.find(el => {
|
|
const t = (el.innerText || '').trim();
|
|
return t.includes(l2Text) && t.length < l2Text.length + 15;
|
|
});
|
|
}
|
|
|
|
if (match) {
|
|
match.scrollIntoView({ behavior: 'instant', block: 'center' });
|
|
await new Promise(r => setTimeout(r, 150));
|
|
match.click();
|
|
return { found: true };
|
|
}
|
|
|
|
if (sidebar) {
|
|
sidebar.scrollBy({ top: 100, behavior: 'instant' });
|
|
await new Promise(r => setTimeout(r, 120));
|
|
}
|
|
}
|
|
return { found: false };
|
|
},
|
|
{ l2Text: level2 }
|
|
);
|
|
|
|
if (!l2Found.found) {
|
|
console.log(C.red(` [NAV] L2 "${level2}" not found under "${level1}".`));
|
|
return false;
|
|
}
|
|
await sleep(2000); // Wait for page load after L2 click
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* navigateViaMenu with automatic retry (up to 2 attempts).
|
|
*/
|
|
async function navigateViaMenuWithRetry(page, level1, level2, maxRetries = 2) {
|
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
const ok = await navigateViaMenu(page, level1, level2);
|
|
if (ok) return true;
|
|
|
|
if (attempt < maxRetries) {
|
|
console.log(C.yellow(` [NAV] Retry ${attempt}/${maxRetries - 1} for ${level1} > ${level2}`));
|
|
// Go back to dashboard and try again
|
|
try {
|
|
await page.goto(DASHBOARD_URL, { waitUntil: 'load', timeout: 12000 });
|
|
await sleep(1500);
|
|
await ensureSidebarExpanded(page);
|
|
await waitForSidebarReady(page, 6000);
|
|
} catch (e) {
|
|
await sleep(2000);
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// ─── Dashboard Navigation ───────────────────────────────────
|
|
|
|
async function ensureLoggedIn(page) {
|
|
const url = page.url();
|
|
if (url.includes('/login')) {
|
|
// Need to re-login
|
|
await page.fill('#userId', AUTH.username);
|
|
await page.fill('#password', AUTH.password);
|
|
await page.click("button[type='submit']");
|
|
await sleep(3000);
|
|
}
|
|
}
|
|
|
|
async function goToDashboard(page) {
|
|
// Force sidebar expanded state BEFORE navigation so page loads with full menu
|
|
try {
|
|
await page.evaluate(() => {
|
|
try {
|
|
const raw = localStorage.getItem('sam-menu');
|
|
if (raw) {
|
|
const data = JSON.parse(raw);
|
|
if (data.state) {
|
|
data.state.sidebarCollapsed = false;
|
|
localStorage.setItem('sam-menu', JSON.stringify(data));
|
|
}
|
|
}
|
|
} catch (e) {}
|
|
});
|
|
} catch (e) { /* page may not be ready yet */ }
|
|
|
|
// Attempt 1: Navigate to dashboard
|
|
try {
|
|
await page.goto(DASHBOARD_URL, { waitUntil: 'load', timeout: 15000 });
|
|
await sleep(1000);
|
|
await ensureLoggedIn(page);
|
|
await ensureSidebarExpanded(page);
|
|
await waitForSidebarReady(page, 8000);
|
|
return;
|
|
} catch (e) {
|
|
const dbgUrl = page.url();
|
|
console.log(C.yellow(` [DASH] attempt 1 failed. URL: ${dbgUrl}, err: ${e.message?.substring(0, 80)}`));
|
|
}
|
|
|
|
// Attempt 2: Reload to force fresh render
|
|
try {
|
|
await page.reload({ waitUntil: 'load', timeout: 10000 });
|
|
await sleep(1500);
|
|
await ensureLoggedIn(page);
|
|
await ensureSidebarExpanded(page);
|
|
await waitForSidebarReady(page, 8000);
|
|
return;
|
|
} catch (e) {
|
|
const dbgUrl = page.url();
|
|
console.log(C.yellow(` [DASH] reload failed. URL: ${dbgUrl}, err: ${e.message?.substring(0, 80)}`));
|
|
}
|
|
|
|
// Attempt 3: Full re-login
|
|
try {
|
|
await page.goto(`${BASE_URL}/ko/login`, { waitUntil: 'load', timeout: 10000 });
|
|
await sleep(500);
|
|
try {
|
|
await page.fill('#userId', AUTH.username);
|
|
await page.fill('#password', AUTH.password);
|
|
await page.click("button[type='submit']");
|
|
await sleep(3000);
|
|
} catch (loginErr) { /* may already be on dashboard */ }
|
|
await ensureSidebarExpanded(page);
|
|
await waitForSidebarReady(page, 8000);
|
|
} catch (e) {
|
|
await sleep(2000);
|
|
}
|
|
}
|
|
|
|
const SCENARIO_TIMEOUT = 180000; // 3 minutes per scenario (batch-create needs extra time)
|
|
const WORKFLOW_TIMEOUT = 300000; // 5 minutes for workflow scenarios (multi-module chains)
|
|
const PERFORMANCE_TIMEOUT = 120000; // 2 minutes for performance scenarios
|
|
|
|
/** Determine timeout based on scenario category */
|
|
function getScenarioTimeout(filename) {
|
|
if (filename.startsWith('workflow-')) return WORKFLOW_TIMEOUT;
|
|
if (filename.startsWith('perf-')) return PERFORMANCE_TIMEOUT;
|
|
return SCENARIO_TIMEOUT;
|
|
}
|
|
|
|
/** Classify scenario into a category for summary grouping */
|
|
function getScenarioCategory(filename) {
|
|
if (filename.startsWith('workflow-')) return 'workflow';
|
|
if (filename.startsWith('perf-')) return 'performance';
|
|
if (filename.startsWith('edge-')) return 'edge-case';
|
|
if (filename.startsWith('a11y-')) return 'accessibility';
|
|
return 'functional';
|
|
}
|
|
|
|
// ─── Page Health Verify ─────────────────────────────────────
|
|
|
|
/**
|
|
* Pre-flight page health check.
|
|
* Runs AFTER menu navigation, BEFORE step execution.
|
|
* Detects page crashes, console errors, Error Boundaries, API failures.
|
|
*
|
|
* Returns: { healthy: boolean, diagnosis: { ... } }
|
|
*/
|
|
async function verifyPageHealth(page, timeout = 8000) {
|
|
const diagnosis = {
|
|
healthy: true,
|
|
url: '',
|
|
crashed: false,
|
|
errorBoundary: null,
|
|
consoleErrors: [],
|
|
apiErrors: [],
|
|
emptySelectValues: 0,
|
|
blankPage: false,
|
|
loadTimeout: false,
|
|
};
|
|
|
|
try {
|
|
diagnosis.url = page.url();
|
|
|
|
// 1. Collect console errors captured during page load
|
|
// (we attach listener before navigation in runScenario, store them on page object)
|
|
diagnosis.consoleErrors = (page.__e2e_console_errors || []).slice(-10);
|
|
|
|
// 2. Check for Error Boundary / crash screen / blank page
|
|
const pageState = await Promise.race([
|
|
page.evaluate(() => {
|
|
const bodyText = document.body?.innerText || '';
|
|
|
|
// Error Boundary patterns (React)
|
|
// NOTE: bodyText(innerText)만 사용. innerHTML은 i18n 번역 JSON에
|
|
// "서버 오류가 발생했습니다" 등이 포함되어 false positive 발생함
|
|
const errorBoundaryPatterns = [
|
|
'오류가 발생했습니다', '일시적인 오류',
|
|
'Something went wrong', 'Error boundary',
|
|
'An error occurred', 'Unhandled Runtime Error',
|
|
'Application error',
|
|
];
|
|
const foundErrorText = errorBoundaryPatterns.find(p =>
|
|
bodyText.includes(p)
|
|
);
|
|
|
|
// Blank page detection
|
|
const hasContent = bodyText.trim().length > 50;
|
|
const hasMainContent = !!document.querySelector(
|
|
'table, [class*="content"], main, [role="main"], [class*="page"], [class*="list"]'
|
|
);
|
|
|
|
// API errors (from step-executor ApiMonitor)
|
|
const apiErrors = (window.__E2E__ ? window.__E2E__.getApiLogs().errors : []).filter(e =>
|
|
e.status >= 400 || e.error
|
|
);
|
|
|
|
// Empty Select.Item values (the exact bug pattern)
|
|
const emptySelectItems = document.querySelectorAll(
|
|
'[role="option"][data-value=""], select option[value=""]'
|
|
);
|
|
|
|
// Check for React error overlay (dev mode)
|
|
const reactErrorOverlay = document.querySelector(
|
|
'[data-nextjs-dialog], #__next-build-error, [class*="nextjs-container-errors"]'
|
|
);
|
|
|
|
return {
|
|
foundErrorText,
|
|
hasContent,
|
|
hasMainContent,
|
|
apiErrorCount: apiErrors.length,
|
|
apiErrors: apiErrors.slice(0, 5).map(e => ({
|
|
url: (e.url || '').substring(0, 100),
|
|
status: e.status,
|
|
method: e.method,
|
|
error: e.error,
|
|
})),
|
|
emptySelectItems: emptySelectItems.length,
|
|
reactErrorOverlay: !!reactErrorOverlay,
|
|
reactErrorMsg: reactErrorOverlay?.innerText?.substring(0, 200) || null,
|
|
};
|
|
}),
|
|
sleep(timeout).then(() => ({ timeout: true })),
|
|
]);
|
|
|
|
if (pageState.timeout) {
|
|
diagnosis.healthy = false;
|
|
diagnosis.loadTimeout = true;
|
|
return diagnosis;
|
|
}
|
|
|
|
// Error Boundary detected
|
|
if (pageState.foundErrorText) {
|
|
diagnosis.healthy = false;
|
|
diagnosis.crashed = true;
|
|
diagnosis.errorBoundary = pageState.foundErrorText;
|
|
}
|
|
|
|
// React error overlay (dev/next.js)
|
|
if (pageState.reactErrorOverlay) {
|
|
diagnosis.healthy = false;
|
|
diagnosis.crashed = true;
|
|
diagnosis.errorBoundary = pageState.reactErrorMsg || 'React Error Overlay detected';
|
|
}
|
|
|
|
// Blank page
|
|
if (!pageState.hasContent && !pageState.hasMainContent) {
|
|
diagnosis.healthy = false;
|
|
diagnosis.blankPage = true;
|
|
}
|
|
|
|
// API errors
|
|
if (pageState.apiErrorCount > 0) {
|
|
diagnosis.apiErrors = pageState.apiErrors;
|
|
// API 500 errors make the page unhealthy
|
|
if (pageState.apiErrors.some(e => e.status >= 500)) {
|
|
diagnosis.healthy = false;
|
|
}
|
|
}
|
|
|
|
// Empty Select values (Radix UI crash pattern)
|
|
diagnosis.emptySelectValues = pageState.emptySelectItems;
|
|
if (pageState.emptySelectItems > 0) {
|
|
diagnosis.healthy = false;
|
|
}
|
|
|
|
// Console errors make it unhealthy if they contain crash indicators
|
|
const criticalConsolePatterns = [
|
|
'Select.Item', 'must have a value', 'Uncaught', 'ChunkLoadError',
|
|
'Cannot read properties of null', 'Cannot read properties of undefined',
|
|
'Maximum update depth exceeded', 'Minified React error',
|
|
];
|
|
const criticalConsoleErrors = diagnosis.consoleErrors.filter(msg =>
|
|
criticalConsolePatterns.some(p => msg.includes(p))
|
|
);
|
|
if (criticalConsoleErrors.length > 0) {
|
|
diagnosis.healthy = false;
|
|
}
|
|
} catch (err) {
|
|
diagnosis.healthy = false;
|
|
diagnosis.crashed = true;
|
|
diagnosis.errorBoundary = `Health check error: ${err.message}`;
|
|
}
|
|
|
|
return diagnosis;
|
|
}
|
|
|
|
/**
|
|
* Post-failure diagnosis.
|
|
* Runs AFTER a scenario fails to collect detailed root cause information.
|
|
* Captures: console errors, DOM state, API logs, screenshot.
|
|
*
|
|
* Returns: { rootCause: string, details: { ... } }
|
|
*/
|
|
async function diagnoseFail(page, result, scenarioId) {
|
|
const diag = {
|
|
rootCause: 'unknown',
|
|
consoleErrors: [],
|
|
apiErrors: [],
|
|
pageState: null,
|
|
screenshotPath: null,
|
|
recommendations: [],
|
|
};
|
|
|
|
try {
|
|
// 1. Capture screenshot
|
|
const ssName = `diag_${scenarioId}_${getTimestamp()}.png`;
|
|
const ssPath = path.join(SCREENSHOTS_DIR, ssName);
|
|
try {
|
|
await page.screenshot({ path: ssPath, fullPage: false, timeout: 5000 });
|
|
diag.screenshotPath = ssPath;
|
|
} catch (e) { /* screenshot failed, continue */ }
|
|
|
|
// 2. Collect console errors
|
|
diag.consoleErrors = (page.__e2e_console_errors || []).slice(-20);
|
|
|
|
// 3. Collect page state & API errors
|
|
try {
|
|
const state = await Promise.race([
|
|
page.evaluate(() => {
|
|
const bodyText = document.body?.innerText || '';
|
|
|
|
// Error Boundary check
|
|
const errorPatterns = [
|
|
'오류가 발생했습니다', '일시적인 오류',
|
|
'Something went wrong', 'Error boundary',
|
|
];
|
|
const errorBoundary = errorPatterns.find(p => bodyText.includes(p));
|
|
|
|
// API errors from step-executor monitor
|
|
const apiData = window.__E2E__ ? window.__E2E__.getApiLogs() : { logs: [], errors: [] };
|
|
const apiLogs = apiData.logs || [];
|
|
const apiErrors = (apiData.errors || []).slice(0, 10).map(e => ({
|
|
url: (e.url || '').substring(0, 120),
|
|
status: e.status,
|
|
method: e.method,
|
|
error: e.error,
|
|
}));
|
|
|
|
// Null data detection in rendered content
|
|
const nullPatterns = bodyText.match(/null|undefined/gi) || [];
|
|
|
|
// DOM stats
|
|
const domNodes = document.getElementsByTagName('*').length;
|
|
const tables = document.querySelectorAll('table');
|
|
const tableRowCount = tables.length > 0 ? tables[0].querySelectorAll('tbody tr').length : 0;
|
|
const hasLoadingSpinner = !!document.querySelector(
|
|
'.loading, .spinner, [class*="skeleton"], [class*="loading"], [class*="Skeleton"]'
|
|
);
|
|
|
|
return {
|
|
url: window.location.href,
|
|
errorBoundary,
|
|
apiTotal: apiLogs.length,
|
|
apiErrors,
|
|
nullCount: nullPatterns.length,
|
|
domNodes,
|
|
tableRowCount,
|
|
hasLoadingSpinner,
|
|
visibleText: bodyText.substring(0, 300),
|
|
};
|
|
}),
|
|
sleep(5000).then(() => null),
|
|
]);
|
|
|
|
if (state) {
|
|
diag.pageState = state;
|
|
diag.apiErrors = state.apiErrors;
|
|
|
|
// Classify root cause
|
|
if (state.errorBoundary) {
|
|
diag.rootCause = 'page_crash';
|
|
diag.recommendations.push('페이지 크래시 - Error Boundary 활성화됨. Console 에러 확인 필요');
|
|
} else if (state.apiErrors.some(e => e.status >= 500)) {
|
|
diag.rootCause = 'api_server_error';
|
|
diag.recommendations.push('백엔드 서버 에러 (5xx). 서버 로그 확인 필요');
|
|
} else if (state.apiErrors.some(e => e.status === 401 || e.status === 403)) {
|
|
diag.rootCause = 'auth_error';
|
|
diag.recommendations.push('인증/권한 에러. 세션 만료 가능성');
|
|
} else if (state.hasLoadingSpinner) {
|
|
diag.rootCause = 'infinite_loading';
|
|
diag.recommendations.push('무한 로딩 상태. API 미응답 또는 프론트엔드 상태 관리 버그');
|
|
} else if (state.tableRowCount === 0 && state.apiTotal > 0) {
|
|
diag.rootCause = 'empty_data';
|
|
diag.recommendations.push('API 응답은 있으나 테이블 데이터 없음. 데이터 변환 또는 필터 문제');
|
|
}
|
|
}
|
|
} catch (e) {
|
|
diag.rootCause = 'page_unresponsive';
|
|
diag.recommendations.push('페이지 응답 없음 (evaluate 실패). 네비게이션 에러 또는 크래시');
|
|
}
|
|
|
|
// 4. Console error pattern matching for specific root causes
|
|
const consoleText = diag.consoleErrors.join(' ');
|
|
if (consoleText.includes('Select.Item') || consoleText.includes('must have a value')) {
|
|
diag.rootCause = 'select_empty_value';
|
|
diag.recommendations = ['Radix Select.Item에 빈 value 전달됨. 데이터 transform에서 null/빈값 방어 필요'];
|
|
} else if (consoleText.includes('ChunkLoadError') || consoleText.includes('Loading chunk')) {
|
|
diag.rootCause = 'chunk_load_error';
|
|
diag.recommendations = ['JS 번들 로드 실패. 배포 상태 또는 네트워크 문제'];
|
|
} else if (consoleText.includes('Cannot read properties of null') || consoleText.includes('Cannot read properties of undefined')) {
|
|
diag.rootCause = 'null_reference';
|
|
diag.recommendations = ['Null 참조 에러. API 응답에서 예상치 못한 null 데이터 가능성'];
|
|
} else if (consoleText.includes('Maximum update depth')) {
|
|
diag.rootCause = 'infinite_render_loop';
|
|
diag.recommendations = ['React 무한 렌더 루프. useEffect 의존성 배열 또는 상태 업데이트 로직 확인'];
|
|
}
|
|
|
|
// 5. Check failed step patterns for additional context
|
|
if (result.steps) {
|
|
const failedSteps = result.steps.filter(s => s.status === 'fail');
|
|
const timeoutSteps = failedSteps.filter(s =>
|
|
(s.error || s.details || '').includes('timeout') || (s.error || s.details || '').includes('Timeout')
|
|
);
|
|
if (timeoutSteps.length > 0 && diag.rootCause === 'unknown') {
|
|
diag.rootCause = 'element_timeout';
|
|
diag.recommendations.push('요소 대기 타임아웃. 페이지 로드 지연 또는 셀렉터 불일치');
|
|
}
|
|
}
|
|
|
|
} catch (err) {
|
|
diag.rootCause = 'diagnosis_error';
|
|
diag.recommendations.push(`진단 중 에러: ${err.message}`);
|
|
}
|
|
|
|
return diag;
|
|
}
|
|
|
|
// ─── Scenario Runner ────────────────────────────────────────
|
|
|
|
async function runScenario(page, scenarioPath) {
|
|
const scenarioJson = JSON.parse(fs.readFileSync(scenarioPath, 'utf-8'));
|
|
const { id, name, steps, selectors, menuNavigation } = scenarioJson;
|
|
|
|
const result = {
|
|
id: id || path.basename(scenarioPath, '.json'),
|
|
name: name || id,
|
|
steps: [],
|
|
passed: 0,
|
|
failed: 0,
|
|
warned: 0,
|
|
totalSteps: 0,
|
|
apiSummary: null,
|
|
error: null,
|
|
stoppedReason: 'complete',
|
|
currentUrl: '',
|
|
startTime: Date.now(),
|
|
endTime: 0,
|
|
healthCheck: null, // Page health verification result
|
|
diagnosis: null, // Post-failure diagnosis result
|
|
};
|
|
|
|
// Attach console error listener for this scenario
|
|
page.__e2e_console_errors = [];
|
|
const consoleHandler = (msg) => {
|
|
if (msg.type() === 'error') {
|
|
page.__e2e_console_errors.push(msg.text().substring(0, 500));
|
|
}
|
|
};
|
|
page.on('console', consoleHandler);
|
|
|
|
// Also capture uncaught page errors
|
|
const pageErrorHandler = (err) => {
|
|
page.__e2e_console_errors.push(`[PAGE_ERROR] ${err.message}`.substring(0, 500));
|
|
};
|
|
page.on('pageerror', pageErrorHandler);
|
|
|
|
if (!steps || steps.length === 0) {
|
|
result.error = 'No steps defined';
|
|
result.stoppedReason = 'no_steps';
|
|
result.endTime = Date.now();
|
|
page.removeListener('console', consoleHandler);
|
|
page.removeListener('pageerror', pageErrorHandler);
|
|
return result;
|
|
}
|
|
|
|
try {
|
|
// Navigate to dashboard first
|
|
await goToDashboard(page);
|
|
|
|
// Inject executor
|
|
await injectExecutor(page);
|
|
|
|
// Menu navigation if specified
|
|
if (menuNavigation && menuNavigation.level1) {
|
|
const navOk = await navigateViaMenuWithRetry(page, menuNavigation.level1, menuNavigation.level2);
|
|
if (!navOk) {
|
|
result.error = `Menu navigation failed: ${menuNavigation.level1} > ${menuNavigation.level2}`;
|
|
result.stoppedReason = 'navigation_failed';
|
|
result.endTime = Date.now();
|
|
page.removeListener('console', consoleHandler);
|
|
page.removeListener('pageerror', pageErrorHandler);
|
|
return result;
|
|
}
|
|
// Re-inject after navigation
|
|
await sleep(1000);
|
|
await injectExecutor(page);
|
|
}
|
|
|
|
// ─── Page Health Check (Pre-flight) ───────────────────
|
|
const health = await verifyPageHealth(page);
|
|
result.healthCheck = health;
|
|
|
|
if (!health.healthy) {
|
|
// Page is unhealthy - run diagnosis immediately and abort
|
|
const diag = await diagnoseFail(page, result, result.id);
|
|
result.diagnosis = diag;
|
|
|
|
// Build descriptive error message
|
|
const reasons = [];
|
|
if (health.crashed || health.errorBoundary)
|
|
reasons.push(`페이지 크래시: ${health.errorBoundary}`);
|
|
if (health.blankPage)
|
|
reasons.push('빈 페이지 (콘텐츠 없음)');
|
|
if (health.emptySelectValues > 0)
|
|
reasons.push(`빈 Select 값 ${health.emptySelectValues}개 감지`);
|
|
if (health.loadTimeout)
|
|
reasons.push('페이지 로드 타임아웃');
|
|
if (health.apiErrors.length > 0)
|
|
reasons.push(`API 에러 ${health.apiErrors.length}건 (${health.apiErrors.map(e => `${e.status} ${e.method}`).join(', ')})`);
|
|
|
|
const consoleSnippet = (health.consoleErrors || [])
|
|
.filter(msg => msg.length > 10)
|
|
.slice(0, 3)
|
|
.map(msg => msg.substring(0, 150));
|
|
|
|
result.error = `[HEALTH_CHECK] ${reasons.join(' | ')}`;
|
|
if (consoleSnippet.length > 0) {
|
|
result.error += ` | Console: ${consoleSnippet.join('; ')}`;
|
|
}
|
|
result.stoppedReason = 'health_check_failed';
|
|
|
|
console.log(C.yellow(` [HEALTH] ✘ ${reasons[0] || 'unhealthy'}`));
|
|
if (diag.rootCause !== 'unknown') {
|
|
console.log(C.yellow(` [DIAG] root cause: ${diag.rootCause}`));
|
|
}
|
|
|
|
result.endTime = Date.now();
|
|
page.removeListener('console', consoleHandler);
|
|
page.removeListener('pageerror', pageErrorHandler);
|
|
return result;
|
|
}
|
|
|
|
// Run steps in batches (handling navigation stops)
|
|
let currentIndex = 0;
|
|
let vars = {};
|
|
const allResults = [];
|
|
|
|
while (currentIndex < steps.length) {
|
|
const batch = steps.slice(currentIndex);
|
|
|
|
let batchResult;
|
|
try {
|
|
batchResult = await page.evaluate(
|
|
async ({ batch, vars, selectors }) => {
|
|
const r = await window.__E2E__.runBatch(batch, vars, { selectors: selectors || {} });
|
|
return r;
|
|
},
|
|
{ batch, vars, selectors: selectors || {} }
|
|
);
|
|
} catch (evalErr) {
|
|
// If evaluate fails (page navigated away, etc.), try re-injection
|
|
try {
|
|
await sleep(2000);
|
|
await injectExecutor(page);
|
|
batchResult = await page.evaluate(
|
|
async ({ batch, vars, selectors }) => {
|
|
const r = await window.__E2E__.runBatch(batch, vars, { selectors: selectors || {} });
|
|
return r;
|
|
},
|
|
{ batch, vars, selectors: selectors || {} }
|
|
);
|
|
} catch (retryErr) {
|
|
result.error = `Evaluate failed: ${retryErr.message}`;
|
|
result.stoppedReason = 'evaluate_error';
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Collect results
|
|
if (batchResult.results) {
|
|
allResults.push(...batchResult.results);
|
|
}
|
|
vars = batchResult.variables || vars;
|
|
result.apiSummary = batchResult.apiSummary;
|
|
result.currentUrl = batchResult.currentUrl;
|
|
|
|
// Handle stoppedReason
|
|
if (batchResult.stoppedReason === 'complete') {
|
|
break;
|
|
}
|
|
|
|
if (batchResult.stoppedReason === 'navigation') {
|
|
await sleep(2000);
|
|
try {
|
|
await injectExecutor(page);
|
|
} catch (e) {
|
|
// Page might still be loading
|
|
await sleep(3000);
|
|
try {
|
|
await injectExecutor(page);
|
|
} catch (e2) {
|
|
result.error = `Re-injection failed after navigation: ${e2.message}`;
|
|
result.stoppedReason = 'reinjection_failed';
|
|
break;
|
|
}
|
|
}
|
|
currentIndex += batchResult.stoppedAtIndex;
|
|
continue;
|
|
}
|
|
|
|
if (batchResult.stoppedReason === 'native_required') {
|
|
// Take screenshot
|
|
const ssName = `${result.id}_step${batchResult.stoppedAtIndex}_${getTimestamp()}.png`;
|
|
try {
|
|
await page.screenshot({ path: path.join(SCREENSHOTS_DIR, ssName), fullPage: false });
|
|
} catch (ssErr) {
|
|
// Screenshot failed, continue
|
|
}
|
|
currentIndex += batchResult.stoppedAtIndex + 1;
|
|
// Re-inject in case page changed
|
|
try { await injectExecutor(page); } catch (e) { /* ignore */ }
|
|
continue;
|
|
}
|
|
|
|
if (batchResult.stoppedReason === 'critical_failure') {
|
|
result.stoppedReason = 'critical_failure';
|
|
break;
|
|
}
|
|
|
|
// Unknown stoppedReason
|
|
break;
|
|
}
|
|
|
|
// Aggregate results
|
|
result.steps = allResults;
|
|
result.totalSteps = allResults.length;
|
|
result.passed = allResults.filter((r) => r.status === 'pass').length;
|
|
result.failed = allResults.filter((r) => r.status === 'fail').length;
|
|
result.warned = allResults.filter((r) => r.status === 'warn').length;
|
|
if (result.stoppedReason === 'complete' && result.failed > 0) {
|
|
result.stoppedReason = 'completed_with_failures';
|
|
}
|
|
|
|
// ─── Post-failure Diagnosis ────────────────────────────
|
|
if (result.failed > 0 || result.error) {
|
|
try {
|
|
const diag = await diagnoseFail(page, result, result.id);
|
|
result.diagnosis = diag;
|
|
if (diag.rootCause !== 'unknown') {
|
|
console.log(C.yellow(` [DIAG] root cause: ${diag.rootCause}`));
|
|
}
|
|
} catch (diagErr) {
|
|
// Diagnosis itself failed, don't block
|
|
}
|
|
}
|
|
} catch (err) {
|
|
result.error = err.message;
|
|
result.stoppedReason = 'exception';
|
|
// Try diagnosis even on exception
|
|
try {
|
|
const diag = await diagnoseFail(page, result, result.id);
|
|
result.diagnosis = diag;
|
|
} catch (diagErr) { /* ignore */ }
|
|
}
|
|
|
|
// Cleanup event listeners
|
|
page.removeListener('console', consoleHandler);
|
|
page.removeListener('pageerror', pageErrorHandler);
|
|
|
|
result.endTime = Date.now();
|
|
return result;
|
|
}
|
|
|
|
// ─── Report Generation ──────────────────────────────────────
|
|
|
|
function generateReport(result, timestamp) {
|
|
const duration = ((result.endTime - result.startTime) / 1000).toFixed(1);
|
|
const hasFail = result.failed > 0 || result.error;
|
|
const status = hasFail ? 'FAIL' : 'PASS';
|
|
const icon = hasFail ? '❌' : '✅';
|
|
|
|
const stepsTable = result.steps
|
|
.map((s) => {
|
|
const statusIcon = s.status === 'pass' ? '✅' : s.status === 'fail' ? '❌' : '⚠️';
|
|
const phase = s.phase || '-';
|
|
const details = (s.details || '').substring(0, 80).replace(/\|/g, '/');
|
|
return `| ${s.stepId} | ${s.name} | ${phase} | ${statusIcon} | ${s.duration}ms | ${details} |`;
|
|
})
|
|
.join('\n');
|
|
|
|
const failedSteps = result.steps.filter((s) => s.status === 'fail');
|
|
const failedTable = failedSteps.length > 0
|
|
? failedSteps
|
|
.map((s) => `| ${s.stepId} | ${s.name} | ${s.phase || '-'} | ${(s.error || s.details || '').substring(0, 100)} |`)
|
|
.join('\n')
|
|
: '';
|
|
|
|
const api = result.apiSummary || { total: 0, success: 0, failed: 0, avgResponseTime: 0, slowCalls: 0 };
|
|
|
|
let md = `# ${icon} E2E 테스트 ${hasFail ? '실패' : '성공'}: ${result.name}
|
|
|
|
**테스트 ID**: ${result.id} | **실행**: ${timestamp} | **결과**: ${status}
|
|
**소요 시간**: ${duration}초${result.error ? ` | **에러**: ${result.error}` : ''}${result.stoppedReason !== 'complete' && result.stoppedReason !== 'completed_with_failures' ? ` | **중단 사유**: ${result.stoppedReason}` : ''}
|
|
|
|
## 테스트 요약
|
|
| 전체 | 성공 | 실패 | 경고 | 성공률 |
|
|
|------|------|------|------|--------|
|
|
| ${result.totalSteps} | ${result.passed} | ${result.failed} | ${result.warned} | ${result.totalSteps > 0 ? Math.round((result.passed / result.totalSteps) * 100) : 0}% |
|
|
`;
|
|
|
|
if (failedTable) {
|
|
md += `
|
|
## 실패 스텝
|
|
| # | 스텝 | Phase | 에러 |
|
|
|---|------|-------|------|
|
|
${failedTable}
|
|
`;
|
|
}
|
|
|
|
md += `
|
|
## 전체 스텝 결과
|
|
| # | 스텝 | Phase | 상태 | 소요시간 | 비고 |
|
|
|---|------|-------|------|---------|------|
|
|
${stepsTable || '| - | (스텝 없음) | - | - | - | - |'}
|
|
|
|
## API 요약
|
|
| 총 호출 | 성공 | 실패 | 평균 응답 | 느린 호출(>2s) |
|
|
|---------|------|------|----------|--------------|
|
|
| ${api.total} | ${api.success} | ${api.failed} | ${api.avgResponseTime}ms | ${api.slowCalls} |
|
|
`;
|
|
|
|
// Health Check section
|
|
if (result.healthCheck) {
|
|
const h = result.healthCheck;
|
|
md += '\n## 페이지 건강 검사\n';
|
|
md += '| 항목 | 결과 |\n|------|------|\n';
|
|
md += '| 상태 | ' + (h.healthy ? '✅ 정상' : '❌ 비정상') + ' |\n';
|
|
md += '| URL | ' + (h.url || '-') + ' |\n';
|
|
if (h.crashed) md += '| 크래시 | ' + (h.errorBoundary || 'Yes') + ' |\n';
|
|
if (h.blankPage) md += '| 빈 페이지 | Yes |\n';
|
|
if (h.loadTimeout) md += '| 로드 타임아웃 | Yes |\n';
|
|
if (h.emptySelectValues > 0) md += '| 빈 Select 값 | ' + h.emptySelectValues + '개 |\n';
|
|
if (h.apiErrors && h.apiErrors.length > 0) {
|
|
const apiErrStr = h.apiErrors.map(function(e) { return e.status + ' ' + e.method + ' ' + e.url; }).join(', ');
|
|
md += '| API 에러 | ' + apiErrStr + ' |\n';
|
|
}
|
|
|
|
if (h.consoleErrors && h.consoleErrors.length > 0) {
|
|
md += '\n### 콘솔 에러 (Health Check)\n';
|
|
h.consoleErrors.slice(0, 5).forEach(function(err, i) {
|
|
md += (i + 1) + '. `' + err.substring(0, 200) + '`\n';
|
|
});
|
|
}
|
|
}
|
|
|
|
// Diagnosis section
|
|
if (result.diagnosis) {
|
|
const d = result.diagnosis;
|
|
md += '\n## 자동 진단\n';
|
|
md += '| 항목 | 내용 |\n|------|------|\n';
|
|
md += '| 근본 원인 | **' + d.rootCause + '** |\n';
|
|
if (d.screenshotPath) md += '| 스크린샷 | ' + path.basename(d.screenshotPath) + ' |\n';
|
|
|
|
if (d.recommendations && d.recommendations.length > 0) {
|
|
md += '\n### 권장 조치\n';
|
|
d.recommendations.forEach(function(rec, i) {
|
|
md += (i + 1) + '. ' + rec + '\n';
|
|
});
|
|
}
|
|
|
|
if (d.consoleErrors && d.consoleErrors.length > 0) {
|
|
md += '\n### 콘솔 에러 (진단)\n';
|
|
d.consoleErrors.slice(0, 10).forEach(function(err, i) {
|
|
md += (i + 1) + '. `' + err.substring(0, 200) + '`\n';
|
|
});
|
|
}
|
|
|
|
if (d.pageState) {
|
|
const ps = d.pageState;
|
|
md += '\n### 페이지 상태\n';
|
|
md += '| 항목 | 값 |\n|------|----|\n';
|
|
md += '| DOM 노드 | ' + (ps.domNodes || '-') + ' |\n';
|
|
md += '| 테이블 행 | ' + (ps.tableRowCount || 0) + ' |\n';
|
|
md += '| API 호출 수 | ' + (ps.apiTotal || 0) + ' |\n';
|
|
md += '| 로딩 스피너 | ' + (ps.hasLoadingSpinner ? 'Yes' : 'No') + ' |\n';
|
|
if (ps.errorBoundary) md += '| Error Boundary | ' + ps.errorBoundary + ' |\n';
|
|
}
|
|
}
|
|
|
|
return md;
|
|
}
|
|
|
|
function saveReport(result, timestamp) {
|
|
const hasFail = result.failed > 0 || result.error;
|
|
|
|
if (hasFail) {
|
|
const filePath = path.join(RESULTS_DIR, `Fail-${result.id}_${timestamp}.md`);
|
|
fs.writeFileSync(filePath, generateReport(result, timestamp), 'utf-8');
|
|
return filePath;
|
|
} else {
|
|
const filePath = path.join(SUCCESS_DIR, `OK-${result.id}_${timestamp}.md`);
|
|
fs.writeFileSync(filePath, generateReport(result, timestamp), 'utf-8');
|
|
return filePath;
|
|
}
|
|
}
|
|
|
|
// ─── Stability Tracking ─────────────────────────────────────
|
|
|
|
/**
|
|
* Read the most recent previous summary report to detect trends.
|
|
* Returns: { prevPassed, prevFailed, prevTotal, prevTimestamp, scenarioResults: Map<id, 'pass'|'fail'> }
|
|
*/
|
|
function getPreviousRunResults() {
|
|
try {
|
|
const summaryFiles = fs.readdirSync(RESULTS_DIR)
|
|
.filter(f => f.startsWith('E2E_FULL_TEST_SUMMARY_') && f.endsWith('.md'))
|
|
.sort()
|
|
.reverse();
|
|
|
|
if (summaryFiles.length === 0) return null;
|
|
|
|
const content = fs.readFileSync(path.join(RESULTS_DIR, summaryFiles[0]), 'utf-8');
|
|
const scenarioResults = new Map();
|
|
|
|
// Parse scenario results from table
|
|
const tableLines = content.split('\n').filter(l => l.startsWith('| ') && /\| [✅❌]/.test(l));
|
|
tableLines.forEach(line => {
|
|
const cols = line.split('|').map(c => c.trim()).filter(Boolean);
|
|
if (cols.length >= 3) {
|
|
// cols: [#, 시나리오명, 결과, ...]
|
|
const name = cols[1];
|
|
const status = cols[2].includes('✅') ? 'pass' : 'fail';
|
|
scenarioResults.set(name, status);
|
|
}
|
|
});
|
|
|
|
// Parse summary stats
|
|
const totalMatch = content.match(/전체 시나리오.*?(\d+)개.*?성공.*?(\d+)개.*?실패.*?(\d+)개/);
|
|
const timeMatch = content.match(/실행 시간.*?:\s*(.+)/);
|
|
|
|
return {
|
|
prevTotal: totalMatch ? parseInt(totalMatch[1]) : 0,
|
|
prevPassed: totalMatch ? parseInt(totalMatch[2]) : 0,
|
|
prevFailed: totalMatch ? parseInt(totalMatch[3]) : 0,
|
|
prevTimestamp: timeMatch ? timeMatch[1].trim() : 'unknown',
|
|
scenarioResults,
|
|
fileName: summaryFiles[0],
|
|
};
|
|
} catch (e) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Detect flaky tests by checking recent OK/Fail report files.
|
|
* A test is flaky if it has both OK- and Fail- reports in the last N runs.
|
|
* Returns: Map<scenarioId, { passCount, failCount }>
|
|
*/
|
|
function detectFlakyTests(lookbackDays = 3) {
|
|
const flakyMap = new Map();
|
|
const cutoff = Date.now() - (lookbackDays * 24 * 60 * 60 * 1000);
|
|
|
|
try {
|
|
// Check success reports
|
|
const okFiles = fs.readdirSync(SUCCESS_DIR).filter(f => f.startsWith('OK-') && f.endsWith('.md'));
|
|
okFiles.forEach(f => {
|
|
const stat = fs.statSync(path.join(SUCCESS_DIR, f));
|
|
if (stat.mtimeMs < cutoff) return;
|
|
const id = f.replace(/^OK-/, '').replace(/_\d{4}-\d{2}-\d{2}.*$/, '');
|
|
if (!flakyMap.has(id)) flakyMap.set(id, { passCount: 0, failCount: 0 });
|
|
flakyMap.get(id).passCount++;
|
|
});
|
|
|
|
// Check fail reports
|
|
const failFiles = fs.readdirSync(RESULTS_DIR).filter(f => f.startsWith('Fail-') && f.endsWith('.md'));
|
|
failFiles.forEach(f => {
|
|
const stat = fs.statSync(path.join(RESULTS_DIR, f));
|
|
if (stat.mtimeMs < cutoff) return;
|
|
const id = f.replace(/^Fail-/, '').replace(/_\d{4}-\d{2}-\d{2}.*$/, '');
|
|
if (!flakyMap.has(id)) flakyMap.set(id, { passCount: 0, failCount: 0 });
|
|
flakyMap.get(id).failCount++;
|
|
});
|
|
} catch (e) { /* ignore */ }
|
|
|
|
// Filter to only include tests that have BOTH passes and failures
|
|
const flaky = new Map();
|
|
for (const [id, counts] of flakyMap) {
|
|
if (counts.passCount > 0 && counts.failCount > 0) {
|
|
flaky.set(id, counts);
|
|
}
|
|
}
|
|
return flaky;
|
|
}
|
|
|
|
// ─── Summary Report ─────────────────────────────────────────
|
|
|
|
function generateSummaryReport(allResults, totalTime, timestamp) {
|
|
const passed = allResults.filter((r) => !r.error && r.failed === 0).length;
|
|
const failed = allResults.length - passed;
|
|
|
|
// Get previous run for trend analysis
|
|
const prevRun = getPreviousRunResults();
|
|
const flakyTests = detectFlakyTests();
|
|
|
|
// Categorize results
|
|
const categories = {};
|
|
allResults.forEach((r) => {
|
|
const cat = getScenarioCategory(r.id || '');
|
|
if (!categories[cat]) categories[cat] = [];
|
|
categories[cat].push(r);
|
|
});
|
|
|
|
// Trend indicators
|
|
const trendIcon = prevRun
|
|
? (passed > prevRun.prevPassed ? '📈' : passed < prevRun.prevPassed ? '📉' : '➡️')
|
|
: '';
|
|
const trendText = prevRun
|
|
? ` ${trendIcon} (이전: ${prevRun.prevPassed}/${prevRun.prevTotal} 성공)`
|
|
: '';
|
|
|
|
let md = `# E2E 전체 테스트 결과 요약
|
|
|
|
**실행 시간**: ${timestamp}
|
|
**총 소요 시간**: ${(totalTime / 1000 / 60).toFixed(1)}분
|
|
**전체 시나리오**: ${allResults.length}개 | **성공**: ${passed}개 | **실패**: ${failed}개${trendText}
|
|
|
|
## 카테고리별 요약
|
|
| 카테고리 | 시나리오 수 | 성공 | 실패 | 성공률 |
|
|
|---------|-----------|------|------|--------|
|
|
`;
|
|
|
|
const catNames = { functional: '기능 테스트', workflow: '비즈니스 워크플로우', performance: '성능 테스트', 'edge-case': '엣지 케이스', accessibility: '접근성 검사' };
|
|
for (const [cat, results] of Object.entries(categories)) {
|
|
const catPassed = results.filter(r => !r.error && r.failed === 0).length;
|
|
const catFailed = results.length - catPassed;
|
|
const rate = results.length > 0 ? Math.round((catPassed / results.length) * 100) : 0;
|
|
md += `| ${catNames[cat] || cat} | ${results.length} | ${catPassed} | ${catFailed} | ${rate}% |\n`;
|
|
}
|
|
|
|
md += `
|
|
## 시나리오별 결과
|
|
| # | 시나리오 | 결과 | 스텝 | 성공 | 실패 | 소요(초) |
|
|
|---|---------|------|------|------|------|---------|
|
|
`;
|
|
|
|
allResults.forEach((r, i) => {
|
|
const hasFail = r.failed > 0 || r.error;
|
|
const icon = hasFail ? '❌' : '✅';
|
|
const duration = ((r.endTime - r.startTime) / 1000).toFixed(1);
|
|
md += `| ${i + 1} | ${r.name} | ${icon} | ${r.totalSteps} | ${r.passed} | ${r.failed} | ${duration} |\n`;
|
|
});
|
|
|
|
// Workflow summary section
|
|
if (categories.workflow && categories.workflow.length > 0) {
|
|
md += `\n## 비즈니스 워크플로우 상세\n`;
|
|
categories.workflow.forEach((r) => {
|
|
const hasFail = r.failed > 0 || r.error;
|
|
const icon = hasFail ? '❌' : '✅';
|
|
const duration = ((r.endTime - r.startTime) / 1000).toFixed(1);
|
|
md += `\n### ${icon} ${r.name}\n`;
|
|
md += `- 스텝: ${r.passed}/${r.totalSteps} 성공 | 소요: ${duration}초\n`;
|
|
if (r.error) md += `- 에러: ${r.error}\n`;
|
|
const phases = r.steps.filter(s => s.phase).map(s => `${s.phase}(${s.status === 'pass' ? '✅' : '❌'})`);
|
|
if (phases.length > 0) md += `- 단계: ${phases.join(' → ')}\n`;
|
|
});
|
|
}
|
|
|
|
// Performance summary section
|
|
if (categories.performance && categories.performance.length > 0) {
|
|
md += `\n## 성능 테스트 요약\n`;
|
|
md += `| 페이지 | 로드 시간 | 등급 | API 평균 | DOM 노드 |\n`;
|
|
md += `|--------|----------|------|---------|----------|\n`;
|
|
categories.performance.forEach((r) => {
|
|
const perfStep = r.steps.find(s => s.phase === 'PERF_MEASURE');
|
|
const perfData = perfStep?.details ? (() => { try { return JSON.parse(perfStep.details); } catch(e) { return null; } })() : null;
|
|
if (perfData) {
|
|
md += `| ${r.name} | ${perfData.loadTime || '-'}ms | ${perfData.grade || '-'} | ${perfData.apiAvg || '-'}ms | ${perfData.domNodes || '-'} |\n`;
|
|
} else {
|
|
md += `| ${r.name} | - | - | - | - |\n`;
|
|
}
|
|
});
|
|
}
|
|
|
|
// Accessibility summary section
|
|
if (categories.accessibility && categories.accessibility.length > 0) {
|
|
md += `\n## 접근성 검사 요약\n`;
|
|
md += `| 페이지 | 점수 | 등급 | Critical | Serious | Moderate |\n`;
|
|
md += `|--------|------|------|----------|---------|----------|\n`;
|
|
categories.accessibility.forEach((r) => {
|
|
const a11yStep = r.steps.find(s => s.phase === 'A11Y_AUDIT');
|
|
const a11yData = a11yStep?.details ? (() => { try { return JSON.parse(a11yStep.details); } catch(e) { return null; } })() : null;
|
|
if (a11yData) {
|
|
md += `| ${r.name} | ${a11yData.score || '-'} | ${a11yData.grade || '-'} | ${a11yData.critical || 0} | ${a11yData.serious || 0} | ${a11yData.moderate || 0} |\n`;
|
|
} else {
|
|
md += `| ${r.name} | - | - | - | - | - |\n`;
|
|
}
|
|
});
|
|
}
|
|
|
|
if (failed > 0) {
|
|
md += `\n## 실패 시나리오 상세\n`;
|
|
allResults
|
|
.filter((r) => r.failed > 0 || r.error)
|
|
.forEach((r) => {
|
|
md += `\n### ❌ ${r.name} (${r.id})\n`;
|
|
if (r.error) md += `- **에러**: ${r.error}\n`;
|
|
|
|
// Diagnosis info
|
|
if (r.diagnosis && r.diagnosis.rootCause !== 'unknown') {
|
|
md += `- **진단**: ${r.diagnosis.rootCause}`;
|
|
if (r.diagnosis.recommendations && r.diagnosis.recommendations.length > 0) {
|
|
md += ` → ${r.diagnosis.recommendations[0]}`;
|
|
}
|
|
md += `\n`;
|
|
}
|
|
|
|
// Health check info
|
|
if (r.healthCheck && !r.healthCheck.healthy) {
|
|
const h = r.healthCheck;
|
|
const issues = [];
|
|
if (h.crashed) issues.push('크래시');
|
|
if (h.blankPage) issues.push('빈 페이지');
|
|
if (h.loadTimeout) issues.push('로드 타임아웃');
|
|
if (h.emptySelectValues > 0) issues.push(`빈 Select 값 ${h.emptySelectValues}개`);
|
|
if (issues.length > 0) md += `- **건강 검사**: ${issues.join(', ')}\n`;
|
|
}
|
|
|
|
const failSteps = r.steps.filter((s) => s.status === 'fail');
|
|
failSteps.forEach((s) => {
|
|
md += `- Step ${s.stepId} (${s.name}): ${s.error || s.details}\n`;
|
|
});
|
|
});
|
|
}
|
|
|
|
// Flaky test detection section
|
|
if (flakyTests.size > 0) {
|
|
md += `\n## ⚠️ 불안정 테스트 (Flaky Tests)\n`;
|
|
md += `최근 3일간 성공과 실패가 모두 발생한 시나리오:\n\n`;
|
|
md += `| 시나리오 | 성공 횟수 | 실패 횟수 | 안정성 |\n`;
|
|
md += `|---------|----------|----------|--------|\n`;
|
|
const sorted = [...flakyTests.entries()].sort((a, b) => b[1].failCount - a[1].failCount);
|
|
sorted.forEach(([id, counts]) => {
|
|
const total = counts.passCount + counts.failCount;
|
|
const stability = Math.round((counts.passCount / total) * 100);
|
|
const grade = stability >= 80 ? '🟡' : stability >= 50 ? '🟠' : '🔴';
|
|
md += `| ${id} | ${counts.passCount} | ${counts.failCount} | ${grade} ${stability}% |\n`;
|
|
});
|
|
}
|
|
|
|
// Trend analysis section
|
|
if (prevRun) {
|
|
md += `\n## 📊 트렌드 분석\n`;
|
|
md += `| 항목 | 이전 실행 | 현재 실행 | 변화 |\n`;
|
|
md += `|------|---------|---------|------|\n`;
|
|
const passedDiff = passed - prevRun.prevPassed;
|
|
const failedDiff = failed - prevRun.prevFailed;
|
|
const passedIcon = passedDiff > 0 ? `📈 +${passedDiff}` : passedDiff < 0 ? `📉 ${passedDiff}` : '➡️ 동일';
|
|
const failedIcon = failedDiff < 0 ? `📈 ${failedDiff}` : failedDiff > 0 ? `📉 +${failedDiff}` : '➡️ 동일';
|
|
md += `| 전체 | ${prevRun.prevTotal} | ${allResults.length} | ${allResults.length - prevRun.prevTotal >= 0 ? '+' : ''}${allResults.length - prevRun.prevTotal} |\n`;
|
|
md += `| 성공 | ${prevRun.prevPassed} | ${passed} | ${passedIcon} |\n`;
|
|
md += `| 실패 | ${prevRun.prevFailed} | ${failed} | ${failedIcon} |\n`;
|
|
md += `\n이전 실행: ${prevRun.prevTimestamp}\n`;
|
|
|
|
// Newly failed / newly passed scenarios
|
|
if (prevRun.scenarioResults.size > 0) {
|
|
const newlyFailed = [];
|
|
const newlyPassed = [];
|
|
allResults.forEach(r => {
|
|
const currentStatus = (!r.error && r.failed === 0) ? 'pass' : 'fail';
|
|
const prevStatus = prevRun.scenarioResults.get(r.name);
|
|
if (prevStatus === 'pass' && currentStatus === 'fail') newlyFailed.push(r.name);
|
|
if (prevStatus === 'fail' && currentStatus === 'pass') newlyPassed.push(r.name);
|
|
});
|
|
if (newlyFailed.length > 0) {
|
|
md += `\n### 🔴 새로 실패한 시나리오 (${newlyFailed.length}개)\n`;
|
|
newlyFailed.forEach(n => md += `- ${n}\n`);
|
|
}
|
|
if (newlyPassed.length > 0) {
|
|
md += `\n### 🟢 새로 통과한 시나리오 (${newlyPassed.length}개)\n`;
|
|
newlyPassed.forEach(n => md += `- ${n}\n`);
|
|
}
|
|
}
|
|
}
|
|
|
|
return md;
|
|
}
|
|
|
|
// ─── Main ───────────────────────────────────────────────────
|
|
|
|
async function main() {
|
|
// --groups: 그룹 목록만 출력하고 종료
|
|
if (LIST_GROUPS) {
|
|
listGroups();
|
|
process.exit(0);
|
|
}
|
|
|
|
// --validate: 시나리오 정합성 검증만 수행하고 종료
|
|
if (VALIDATE_ONLY) {
|
|
let files = fs.readdirSync(SCENARIOS_DIR)
|
|
.filter(f => f.endsWith('.json') && !f.startsWith('_'))
|
|
.sort();
|
|
if (FILTER) files = files.filter(f => f.includes(FILTER));
|
|
if (GROUP) files = filterByGroup(files, GROUP);
|
|
const result = validateScenarios(files);
|
|
process.exit(result.errors > 0 ? 1 : 0);
|
|
}
|
|
|
|
console.log(C.bold('\n=== E2E 전체 테스트 러너 ==='));
|
|
console.log(`서버: ${BASE_URL}`);
|
|
console.log(`모드: ${HEADLESS ? 'headless' : 'headed'}`);
|
|
if (WORKFLOW_ONLY) console.log(`카테고리: workflow only`);
|
|
if (FILTER) console.log(`필터: ${FILTER}`);
|
|
if (GROUP) console.log(`그룹: ${GROUP}`);
|
|
if (EXCLUDE) console.log(`제외: ${EXCLUDE}`);
|
|
if (SKIP_PASSED) console.log(`스킵: 이미 성공한 시나리오 건너뛰기 (--skip-passed)`);
|
|
if (ITERATE) console.log(`반복 모드: 실패 시나리오 자동 재실행 (최대 ${MAX_ITERATIONS}회)`);
|
|
if (STAGE_MODE) console.log(`단계 모드: 카테고리별 순차 실행`);
|
|
if (FAIL_ONLY) console.log(`실패 집중 모드: 이전 실패 시나리오만 실행 (--fail-only)`);
|
|
console.log('');
|
|
|
|
// Ensure directories
|
|
ensureDir(RESULTS_DIR);
|
|
ensureDir(SUCCESS_DIR);
|
|
ensureDir(SCREENSHOTS_DIR);
|
|
|
|
// Collect scenario files (skip disabled scenarios)
|
|
let scenarioFiles = fs.readdirSync(SCENARIOS_DIR)
|
|
.filter((f) => f.endsWith('.json') && !f.startsWith('_'))
|
|
.filter((f) => {
|
|
try {
|
|
const data = JSON.parse(fs.readFileSync(path.join(SCENARIOS_DIR, f), 'utf-8'));
|
|
return data.enabled !== false; // Skip if explicitly disabled
|
|
} catch (e) {
|
|
return true; // Include if can't parse (will fail later with better error)
|
|
}
|
|
})
|
|
.sort();
|
|
|
|
if (WORKFLOW_ONLY) {
|
|
scenarioFiles = scenarioFiles.filter((f) => f.startsWith('workflow-'));
|
|
}
|
|
if (FILTER) {
|
|
scenarioFiles = scenarioFiles.filter((f) => f.includes(FILTER));
|
|
}
|
|
if (EXCLUDE) {
|
|
scenarioFiles = scenarioFiles.filter((f) => !f.includes(EXCLUDE));
|
|
}
|
|
if (GROUP) {
|
|
scenarioFiles = filterByGroup(scenarioFiles, GROUP);
|
|
}
|
|
|
|
// --skip-passed: 이미 성공한 시나리오 건너뛰기
|
|
let skippedCount = 0;
|
|
if (SKIP_PASSED) {
|
|
const passedIds = getPassedScenarioIds();
|
|
const before = scenarioFiles.length;
|
|
scenarioFiles = scenarioFiles.filter((f) => {
|
|
const id = f.replace('.json', '');
|
|
return !passedIds.has(id);
|
|
});
|
|
skippedCount = before - scenarioFiles.length;
|
|
if (skippedCount > 0) {
|
|
console.log(C.cyan(`이미 성공한 시나리오 ${skippedCount}개 건너뜀 (--skip-passed)\n`));
|
|
}
|
|
}
|
|
|
|
// --fail-only: 이전 실패 시나리오만 실행
|
|
if (FAIL_ONLY) {
|
|
const { ids: failedIds, source } = getFailedScenarioIds();
|
|
if (failedIds.size === 0) {
|
|
console.log(C.green('이전 실패 시나리오 없음! 전체 PASS 상태입니다.'));
|
|
process.exit(0);
|
|
}
|
|
const before = scenarioFiles.length;
|
|
scenarioFiles = scenarioFiles.filter((f) => {
|
|
const id = f.replace('.json', '');
|
|
return failedIds.has(id);
|
|
});
|
|
const excluded = before - scenarioFiles.length;
|
|
console.log(C.yellow(`실패 집중 모드: ${failedIds.size}개 실패 시나리오 대상`));
|
|
console.log(C.dim(` 소스: ${source}`));
|
|
console.log(C.dim(` 대상: ${Array.from(failedIds).join(', ')}`));
|
|
console.log(C.dim(` 건너뜀: ${excluded}개 성공 시나리오\n`));
|
|
}
|
|
|
|
const totalScenarios = scenarioFiles.length;
|
|
console.log(`시나리오: ${totalScenarios}개 발견\n`);
|
|
|
|
if (totalScenarios === 0) {
|
|
console.log(C.red('실행할 시나리오가 없습니다.'));
|
|
process.exit(1);
|
|
}
|
|
|
|
// Launch browser
|
|
const browser = await chromium.launch({
|
|
headless: HEADLESS,
|
|
args: [
|
|
`--window-position=1920,0`,
|
|
'--window-size=1920,1080',
|
|
'--disable-blink-features=AutomationControlled',
|
|
],
|
|
});
|
|
|
|
const context = await browser.newContext({
|
|
viewport: { width: 1920, height: 1080 },
|
|
locale: 'ko-KR',
|
|
ignoreHTTPSErrors: true,
|
|
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
|
});
|
|
|
|
const page = await context.newPage();
|
|
|
|
// Mask automation detection
|
|
await page.addInitScript(() => {
|
|
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
|
});
|
|
|
|
// Login
|
|
console.log(C.cyan('로그인 중...'));
|
|
try {
|
|
await page.goto(`${BASE_URL}/ko/login`, { waitUntil: 'domcontentloaded', timeout: 20000 });
|
|
await sleep(1000);
|
|
|
|
await page.fill('#userId', AUTH.username);
|
|
await page.fill('#password', AUTH.password);
|
|
await page.click("button[type='submit']");
|
|
await sleep(3000);
|
|
|
|
const url = page.url();
|
|
if (url.includes('/login')) {
|
|
console.log(C.red('로그인 실패! 대시보드로 이동하지 못함'));
|
|
await browser.close();
|
|
process.exit(1);
|
|
}
|
|
console.log(C.green('로그인 성공!\n'));
|
|
} catch (loginErr) {
|
|
console.log(C.red(`로그인 에러: ${loginErr.message}`));
|
|
await browser.close();
|
|
process.exit(1);
|
|
}
|
|
|
|
// ─── Helper: Run a list of scenario files and return results ────
|
|
async function runScenarioList(files, label, counter) {
|
|
const results = [];
|
|
for (let i = 0; i < files.length; i++) {
|
|
const file = files[i];
|
|
const scenarioPath = path.join(SCENARIOS_DIR, file);
|
|
counter.current++;
|
|
const num = `(${counter.current}/${counter.total})`;
|
|
|
|
process.stdout.write(`${C.dim(num)} ${label ? `[${label}] ` : ''}${file.replace('.json', '')} ... `);
|
|
|
|
let result;
|
|
const timeout = getScenarioTimeout(file);
|
|
try {
|
|
result = await Promise.race([
|
|
runScenario(page, scenarioPath),
|
|
sleep(timeout).then(() => ({
|
|
id: file.replace('.json', ''),
|
|
name: file.replace('.json', ''),
|
|
steps: [],
|
|
passed: 0,
|
|
failed: 0,
|
|
warned: 0,
|
|
totalSteps: 0,
|
|
apiSummary: null,
|
|
error: `Timeout (>${timeout / 1000}s)`,
|
|
stoppedReason: 'timeout',
|
|
currentUrl: '',
|
|
startTime: Date.now() - timeout,
|
|
endTime: Date.now(),
|
|
})),
|
|
]);
|
|
} catch (scenarioErr) {
|
|
result = {
|
|
id: file.replace('.json', ''),
|
|
name: file.replace('.json', ''),
|
|
steps: [],
|
|
passed: 0,
|
|
failed: 0,
|
|
warned: 0,
|
|
totalSteps: 0,
|
|
apiSummary: null,
|
|
error: scenarioErr.message,
|
|
stoppedReason: 'exception',
|
|
currentUrl: '',
|
|
startTime: Date.now(),
|
|
endTime: Date.now(),
|
|
};
|
|
}
|
|
|
|
results.push(result);
|
|
|
|
// Save report
|
|
const ts = getTimestamp();
|
|
saveReport(result, ts);
|
|
|
|
// Console output
|
|
const hasFail = result.failed > 0 || result.error;
|
|
const duration = ((result.endTime - result.startTime) / 1000).toFixed(1);
|
|
if (hasFail) {
|
|
console.log(`${C.red('FAIL')} ${C.dim(`(${result.passed}/${result.totalSteps} passed, ${duration}s)`)}`);
|
|
} else {
|
|
console.log(`${C.green('PASS')} ${C.dim(`(${result.passed}/${result.totalSteps}, ${duration}s)`)}`);
|
|
}
|
|
|
|
// After login scenario or if logged out, re-login
|
|
if (file === 'login.json' || page.url().includes('/login')) {
|
|
try {
|
|
await page.goto(`${BASE_URL}/ko/login`, { waitUntil: 'domcontentloaded', timeout: 10000 });
|
|
await sleep(500);
|
|
await page.fill('#userId', AUTH.username);
|
|
await page.fill('#password', AUTH.password);
|
|
await page.click("button[type='submit']");
|
|
await sleep(3000);
|
|
} catch (reloginErr) {
|
|
// Continue anyway
|
|
}
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
// ─── Stage Mode: Category-by-category execution ────────────
|
|
const STAGE_ORDER = ['accessibility', 'edge-case', 'performance', 'workflow', 'functional'];
|
|
const STAGE_NAMES = {
|
|
'accessibility': '접근성 검사',
|
|
'edge-case': '엣지 케이스',
|
|
'performance': '성능 테스트',
|
|
'workflow': '비즈니스 워크플로우',
|
|
'functional': '기능 테스트',
|
|
};
|
|
|
|
// Run scenarios
|
|
const allResults = [];
|
|
const iterationHistory = []; // Track iteration progress
|
|
const startTime = Date.now();
|
|
|
|
if (STAGE_MODE) {
|
|
// ─── STAGE MODE: Run by category ─────────────────────────
|
|
const categorized = {};
|
|
for (const file of scenarioFiles) {
|
|
const cat = getScenarioCategory(file);
|
|
if (!categorized[cat]) categorized[cat] = [];
|
|
categorized[cat].push(file);
|
|
}
|
|
|
|
let stageNum = 0;
|
|
const counter = { current: 0, total: totalScenarios };
|
|
|
|
for (const cat of STAGE_ORDER) {
|
|
const files = categorized[cat];
|
|
if (!files || files.length === 0) continue;
|
|
|
|
stageNum++;
|
|
const catName = STAGE_NAMES[cat] || cat;
|
|
console.log(C.bold(`\n── Stage ${stageNum}: ${catName} (${files.length}개) ──`));
|
|
|
|
const stageResults = await runScenarioList(files, catName, counter);
|
|
allResults.push(...stageResults);
|
|
|
|
// Stage summary
|
|
const stagePassed = stageResults.filter(r => !r.error && r.failed === 0).length;
|
|
const stageFailed = stageResults.length - stagePassed;
|
|
const stageRate = stageResults.length > 0 ? Math.round((stagePassed / stageResults.length) * 100) : 0;
|
|
|
|
console.log(C.cyan(`\n ▸ ${catName} 결과: ${stagePassed}/${stageResults.length} PASS (${stageRate}%)`));
|
|
if (stageFailed > 0) {
|
|
const failNames = stageResults.filter(r => r.failed > 0 || r.error).map(r => r.id);
|
|
console.log(C.red(` ▸ 실패: ${failNames.join(', ')}`));
|
|
}
|
|
}
|
|
|
|
} else {
|
|
// ─── NORMAL MODE: Run all sequentially ───────────────────
|
|
const counter = { current: 0, total: totalScenarios };
|
|
const results = await runScenarioList(scenarioFiles, '', counter);
|
|
allResults.push(...results);
|
|
}
|
|
|
|
// ─── Iteration 0 (initial run) summary ───────────────────────
|
|
const initialPassed = allResults.filter(r => !r.error && r.failed === 0).length;
|
|
const initialFailed = allResults.length - initialPassed;
|
|
|
|
iterationHistory.push({
|
|
iteration: 0,
|
|
label: '초기 실행',
|
|
total: allResults.length,
|
|
passed: initialPassed,
|
|
failed: initialFailed,
|
|
rate: allResults.length > 0 ? Math.round((initialPassed / allResults.length) * 100) : 0,
|
|
fixed: [],
|
|
stillFailing: allResults.filter(r => r.failed > 0 || r.error).map(r => r.id),
|
|
});
|
|
|
|
// ─── ITERATE MODE: Re-run failed scenarios ─────────────────
|
|
if (ITERATE && initialFailed > 0) {
|
|
for (let iteration = 1; iteration <= MAX_ITERATIONS; iteration++) {
|
|
const failedIds = allResults.filter(r => r.failed > 0 || r.error).map(r => r.id);
|
|
if (failedIds.length === 0) {
|
|
console.log(C.green(C.bold(`\n🎯 반복 ${iteration} 불필요: 모든 시나리오 PASS!`)));
|
|
break;
|
|
}
|
|
|
|
const failedFiles = failedIds.map(id => `${id}.json`).filter(f => fs.existsSync(path.join(SCENARIOS_DIR, f)));
|
|
if (failedFiles.length === 0) break;
|
|
|
|
console.log(C.bold(C.yellow(`\n═══ 반복 ${iteration}/${MAX_ITERATIONS}: ${failedFiles.length}개 실패 시나리오 재실행 ═══`)));
|
|
console.log(C.dim(` 대상: ${failedIds.join(', ')}\n`));
|
|
|
|
const counter = { current: 0, total: failedFiles.length };
|
|
const retryResults = await runScenarioList(failedFiles, `재시도 ${iteration}`, counter);
|
|
|
|
// Update allResults with retry results
|
|
const fixedInThisIteration = [];
|
|
for (const retryResult of retryResults) {
|
|
const idx = allResults.findIndex(r => r.id === retryResult.id);
|
|
if (idx >= 0) {
|
|
const wasFail = allResults[idx].failed > 0 || allResults[idx].error;
|
|
const nowPass = retryResult.failed === 0 && !retryResult.error;
|
|
if (wasFail && nowPass) {
|
|
fixedInThisIteration.push(retryResult.id);
|
|
}
|
|
allResults[idx] = retryResult; // Replace with latest result
|
|
}
|
|
}
|
|
|
|
// Iteration summary
|
|
const iterPassed = allResults.filter(r => !r.error && r.failed === 0).length;
|
|
const iterFailed = allResults.length - iterPassed;
|
|
const stillFailing = allResults.filter(r => r.failed > 0 || r.error).map(r => r.id);
|
|
|
|
iterationHistory.push({
|
|
iteration,
|
|
label: `반복 ${iteration}`,
|
|
total: allResults.length,
|
|
passed: iterPassed,
|
|
failed: iterFailed,
|
|
rate: allResults.length > 0 ? Math.round((iterPassed / allResults.length) * 100) : 0,
|
|
fixed: fixedInThisIteration,
|
|
stillFailing,
|
|
});
|
|
|
|
console.log(C.cyan(`\n ▸ 반복 ${iteration} 결과: ${iterPassed}/${allResults.length} PASS (${Math.round((iterPassed / allResults.length) * 100)}%)`));
|
|
if (fixedInThisIteration.length > 0) {
|
|
console.log(C.green(` ▸ 이번 반복에서 수정됨: ${fixedInThisIteration.join(', ')}`));
|
|
}
|
|
if (stillFailing.length > 0) {
|
|
console.log(C.red(` ▸ 여전히 실패: ${stillFailing.join(', ')}`));
|
|
} else {
|
|
console.log(C.green(C.bold(`\n🎯 모든 시나리오 PASS 달성! (반복 ${iteration}회 만에)`)));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
const totalTime = Date.now() - startTime;
|
|
|
|
// Generate summary report (with iteration history)
|
|
const summaryTs = getTimestamp();
|
|
let summaryMd = generateSummaryReport(allResults, totalTime, summaryTs);
|
|
|
|
// Append iteration history to summary if iterate mode was used
|
|
if (ITERATE && iterationHistory.length > 1) {
|
|
summaryMd += `\n## 반복 실행 이력\n`;
|
|
summaryMd += `| 반복 | 전체 | 성공 | 실패 | 성공률 | 수정됨 | 여전히 실패 |\n`;
|
|
summaryMd += `|:----:|:----:|:----:|:----:|:------:|--------|------------|\n`;
|
|
for (const h of iterationHistory) {
|
|
const fixedStr = h.fixed.length > 0 ? h.fixed.join(', ') : '-';
|
|
const failStr = h.stillFailing.length > 0 ? h.stillFailing.join(', ') : '-';
|
|
summaryMd += `| ${h.label} | ${h.total} | ${h.passed} | ${h.failed} | ${h.rate}% | ${fixedStr} | ${failStr} |\n`;
|
|
}
|
|
|
|
// Progress bar
|
|
summaryMd += `\n### 반복별 추이\n\`\`\`\n`;
|
|
for (const h of iterationHistory) {
|
|
const barLen = 40;
|
|
const filled = Math.round((h.rate / 100) * barLen);
|
|
const bar = '█'.repeat(filled) + '░'.repeat(barLen - filled);
|
|
summaryMd += `${h.label.padEnd(12)} ${bar} ${h.rate}% (${h.passed}/${h.total})\n`;
|
|
}
|
|
summaryMd += `\`\`\`\n`;
|
|
}
|
|
|
|
// Append stage summary if stage mode was used
|
|
if (STAGE_MODE) {
|
|
summaryMd += `\n## 단계별 실행 순서\n`;
|
|
summaryMd += `실행 순서: ${STAGE_ORDER.filter(c => allResults.some(r => getScenarioCategory(r.id) === c)).map(c => STAGE_NAMES[c] || c).join(' → ')}\n`;
|
|
}
|
|
|
|
const summaryPath = path.join(RESULTS_DIR, `E2E_FULL_TEST_SUMMARY_${summaryTs}.md`);
|
|
fs.writeFileSync(summaryPath, summaryMd, 'utf-8');
|
|
|
|
// Close browser
|
|
await browser.close();
|
|
|
|
// Print summary
|
|
const passCount = allResults.filter((r) => !r.error && r.failed === 0).length;
|
|
const failCount = allResults.length - passCount;
|
|
|
|
console.log(C.bold('\n=== 테스트 완료 ==='));
|
|
if (FAIL_ONLY) console.log(C.yellow(`[실패 집중 모드] 이전 실패 시나리오 ${totalScenarios}개만 실행`));
|
|
console.log(`전체: ${totalScenarios} | ${C.green(`성공: ${passCount}`)} | ${failCount > 0 ? C.red(`실패: ${failCount}`) : '실패: 0'}`);
|
|
if (ITERATE && iterationHistory.length > 1) {
|
|
const lastIter = iterationHistory[iterationHistory.length - 1];
|
|
const firstIter = iterationHistory[0];
|
|
const improvement = lastIter.passed - firstIter.passed;
|
|
if (improvement > 0) {
|
|
console.log(C.green(`반복 개선: +${improvement}개 PASS (${firstIter.rate}% → ${lastIter.rate}%)`));
|
|
}
|
|
console.log(`반복 횟수: ${iterationHistory.length - 1}회`);
|
|
}
|
|
console.log(`소요 시간: ${(totalTime / 1000 / 60).toFixed(1)}분`);
|
|
console.log(`요약 리포트: ${summaryPath}`);
|
|
console.log('');
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error(C.red(`\n치명적 에러: ${err.message}`));
|
|
console.error(err.stack);
|
|
process.exit(1);
|
|
});
|