feat: E2E 테스트 러너/엔진 고급 기능 강화
step-executor.js: - wait_for_modal: interactive element 대기 옵션 추가 - wait_for_dialog_ready: 새 액션 (입력필드/버튼 렌더링 대기) - retryAction: progressive delay (500→1000→1500ms) + DOM context 진단 run-all.js: - --validate: 시나리오 JSON dry-run 검증 플래그 - verifyPageHealth(): 페이지 사전 건강성 체크 - diagnoseFail(): 실패 원인 자동 진단 - getPreviousRunResults(): 이전 실행 결과 파싱 - detectFlakyTests(): 3일간 flaky 테스트 감지 - generateSummaryReport(): 트렌드 분석/비교 기능 추가 전체 209/209 ALL PASS 검증 완료 (89.8분) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
김보곤
@@ -18,6 +18,10 @@
|
||||
* node e2e/runner/run-all.js --stage # 카테고리별 단계 실행 (accessibility → edge → perf → workflow → functional)
|
||||
* node e2e/runner/run-all.js --fail-only # 최근 요약 리포트의 실패 시나리오만 실행
|
||||
* node e2e/runner/run-all.js --fail-only --iterate # 실패 시나리오만 반복 실행
|
||||
* node e2e/runner/run-all.js --group accounting # 모듈 그룹 실행 (회계관리)
|
||||
* node e2e/runner/run-all.js --group hr,approval # 여러 그룹 조합 (쉼표 구분)
|
||||
* node e2e/runner/run-all.js --group search # 테스트 유형 그룹 (검색 테스트)
|
||||
* node e2e/runner/run-all.js --groups # 사용 가능한 그룹 목록 출력
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
@@ -62,6 +66,90 @@ const MAX_ITERATIONS = (() => {
|
||||
})();
|
||||
const STAGE_MODE = args.includes('--stage');
|
||||
const FAIL_ONLY = args.includes('--fail-only');
|
||||
const GROUP = (() => {
|
||||
const idx = args.indexOf('--group');
|
||||
return idx >= 0 && args[idx + 1] ? args[idx + 1] : null;
|
||||
})();
|
||||
const LIST_GROUPS = args.includes('--groups');
|
||||
const VALIDATE_ONLY = args.includes('--validate');
|
||||
|
||||
// ─── Group Definitions ──────────────────────────────────────
|
||||
|
||||
// 테스트 유형 그룹 (파일명 prefix 매칭)
|
||||
const TYPE_GROUPS = {
|
||||
search: ['search-'],
|
||||
a11y: ['a11y-'],
|
||||
perf: ['perf-'],
|
||||
edge: ['edge-'],
|
||||
crud: ['full-crud-', 'create-delete-'],
|
||||
workflow: ['workflow-'],
|
||||
input: ['input-fields-'],
|
||||
batch: ['batch-'],
|
||||
reload: ['reload-persist-'],
|
||||
detail: ['detail-'],
|
||||
pagination: ['pagination-sort-'],
|
||||
api: ['api-health-'],
|
||||
validation: ['form-validation-'],
|
||||
};
|
||||
|
||||
// 모듈 그룹 (menuNavigation.level1 매칭)
|
||||
const MODULE_GROUPS = {
|
||||
accounting: ['회계관리'],
|
||||
hr: ['인사관리'],
|
||||
sales: ['판매관리'],
|
||||
board: ['게시판'],
|
||||
settings: ['설정'],
|
||||
production: ['생산관리'],
|
||||
material: ['자재관리'],
|
||||
approval: ['결재관리'],
|
||||
quality: ['품질관리'],
|
||||
customer: ['고객센터'],
|
||||
purchase: ['구매관리'],
|
||||
standard: ['기준정보 관리'],
|
||||
shipping: ['출고관리'],
|
||||
item: ['품목관리'],
|
||||
dashboard: ['시스템 대시보드'],
|
||||
};
|
||||
|
||||
/**
|
||||
* --group 필터링: 모듈 그룹(level1) 또는 테스트 유형 그룹(파일명 prefix) 매칭
|
||||
*/
|
||||
function filterByGroup(files, groupStr) {
|
||||
const groupNames = groupStr.split(',').map(g => g.trim().toLowerCase());
|
||||
// 잘못된 그룹명 경고
|
||||
const invalidGroups = groupNames.filter(g => !TYPE_GROUPS[g] && !MODULE_GROUPS[g]);
|
||||
if (invalidGroups.length > 0) {
|
||||
const allGroupNames = [...Object.keys(TYPE_GROUPS), ...Object.keys(MODULE_GROUPS)].sort();
|
||||
console.log(C.red(`알 수 없는 그룹: ${invalidGroups.join(', ')}`));
|
||||
console.log(C.dim(`사용 가능한 그룹: ${allGroupNames.join(', ')}`));
|
||||
console.log(C.dim(`전체 목록: node e2e/runner/run-all.js --groups\n`));
|
||||
}
|
||||
// 모듈 그룹이 포함된 경우 level1 캐시를 한 번에 구축
|
||||
const needsLevel1 = groupNames.some(g => MODULE_GROUPS[g]);
|
||||
const level1Cache = {};
|
||||
if (needsLevel1) {
|
||||
for (const f of files) {
|
||||
try {
|
||||
const data = JSON.parse(fs.readFileSync(path.join(SCENARIOS_DIR, f), 'utf-8'));
|
||||
level1Cache[f] = data.menuNavigation?.level1 || null;
|
||||
} catch { level1Cache[f] = null; }
|
||||
}
|
||||
}
|
||||
return files.filter(f => {
|
||||
const basename = f.replace('.json', '');
|
||||
return groupNames.some(groupName => {
|
||||
// 테스트 유형 그룹 (prefix 매칭)
|
||||
if (TYPE_GROUPS[groupName]) {
|
||||
return TYPE_GROUPS[groupName].some(prefix => basename.startsWith(prefix));
|
||||
}
|
||||
// 모듈 그룹 (level1 매칭)
|
||||
if (MODULE_GROUPS[groupName]) {
|
||||
return MODULE_GROUPS[groupName].includes(level1Cache[f]);
|
||||
}
|
||||
return false;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Helpers ────────────────────────────────────────────────
|
||||
|
||||
@@ -170,6 +258,246 @@ const C = {
|
||||
bold: (s) => `\x1b[1m${s}\x1b[0m`,
|
||||
};
|
||||
|
||||
/**
|
||||
* --groups: 사용 가능한 그룹 목록 + 시나리오 수 출력
|
||||
*/
|
||||
function listGroups() {
|
||||
const allFiles = fs.readdirSync(SCENARIOS_DIR)
|
||||
.filter(f => f.endsWith('.json') && !f.startsWith('_'))
|
||||
.sort();
|
||||
|
||||
// level1 캐시 구축
|
||||
const level1Cache = {};
|
||||
for (const f of allFiles) {
|
||||
try {
|
||||
const data = JSON.parse(fs.readFileSync(path.join(SCENARIOS_DIR, f), 'utf-8'));
|
||||
level1Cache[f] = data.menuNavigation?.level1 || null;
|
||||
} catch { level1Cache[f] = null; }
|
||||
}
|
||||
|
||||
console.log(C.bold('\n=== 사용 가능한 그룹 목록 ===\n'));
|
||||
|
||||
console.log(C.cyan(C.bold('모듈 그룹 (menuNavigation.level1 기반):')));
|
||||
const moduleEntries = Object.entries(MODULE_GROUPS).sort((a, b) => {
|
||||
const countA = allFiles.filter(f => a[1].includes(level1Cache[f])).length;
|
||||
const countB = allFiles.filter(f => b[1].includes(level1Cache[f])).length;
|
||||
return countB - countA;
|
||||
});
|
||||
for (const [name, level1s] of moduleEntries) {
|
||||
const count = allFiles.filter(f => level1s.includes(level1Cache[f])).length;
|
||||
if (count > 0) {
|
||||
console.log(` ${C.green(name.padEnd(14))} ${String(count).padStart(3)}개 (${level1s.join(', ')})`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('');
|
||||
console.log(C.cyan(C.bold('테스트 유형 그룹 (파일명 prefix 기반):')));
|
||||
const typeEntries = Object.entries(TYPE_GROUPS).sort((a, b) => {
|
||||
const countA = allFiles.filter(f => a[1].some(p => f.replace('.json', '').startsWith(p))).length;
|
||||
const countB = allFiles.filter(f => b[1].some(p => f.replace('.json', '').startsWith(p))).length;
|
||||
return countB - countA;
|
||||
});
|
||||
for (const [name, prefixes] of typeEntries) {
|
||||
const count = allFiles.filter(f => prefixes.some(p => f.replace('.json', '').startsWith(p))).length;
|
||||
if (count > 0) {
|
||||
console.log(` ${C.green(name.padEnd(14))} ${String(count).padStart(3)}개 (${prefixes.join(', ')})`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n${C.dim('사용법: node e2e/runner/run-all.js --group <그룹명>[,<그룹명>...]')}`);
|
||||
console.log(`${C.dim(' 예: --group accounting --group hr,approval --group search')}\n`);
|
||||
}
|
||||
|
||||
// ─── Scenario Validation ────────────────────────────────────
|
||||
|
||||
/**
|
||||
* --validate: Dry-run scenario validation without browser.
|
||||
* Checks JSON structure, required fields, deprecated patterns, and action validity.
|
||||
*/
|
||||
function validateScenarios(files) {
|
||||
const VALID_ACTIONS = new Set([
|
||||
'click', 'click_nth', 'click_row', 'click_first_row', 'click_button',
|
||||
'click_dialog_confirm', 'click_and_confirm', 'click_if_exists',
|
||||
'fill', 'fill_nth', 'fill_form', 'fill_and_wait', 'clear', 'edit_field',
|
||||
'select', 'select_dropdown', 'select_filter',
|
||||
'check', 'uncheck', 'check_nth', 'uncheck_nth',
|
||||
'wait', 'wait_for_element', 'wait_for_table', 'wait_for_modal', 'wait_for_navigation',
|
||||
'verify_element', 'verify_checks', 'verify_text', 'verify_url', 'verify_url_stability',
|
||||
'verify_table', 'verify_table_structure', 'verify_data', 'verify_detail',
|
||||
'verify_not_mockup', 'verify_dialog', 'verify_input_value', 'verify_page',
|
||||
'verify_console', 'verify_data_change', 'verify_edit_mode', 'verify_toast',
|
||||
'save_url', 'extract_from_url', 'capture', 'close_modal', 'close_modal_if_open',
|
||||
'scrollAndFind', 'evaluate', 'search', 'press_key', 'blur', 'hover',
|
||||
'generate_timestamp', 'navigate', 'reload', 'navigate_back', 'menu_navigate',
|
||||
'workflow_context_save', 'workflow_context_load', 'cross_module_verify',
|
||||
'measure_performance', 'measure_api_performance', 'assert_performance',
|
||||
'fill_boundary', 'rapid_click', 'accessibility_audit', 'keyboard_navigate',
|
||||
]);
|
||||
|
||||
// Common aliases that should also be accepted
|
||||
const VALID_ALIASES = new Set([
|
||||
// Fill aliases
|
||||
'type', 'input', 'text', 'textarea', 'email', 'password', 'number',
|
||||
'clear_and_type', 'clear-and-type', 'change_date', 'change_date_range',
|
||||
'date_range', 'date', 'datepicker', 'setDateRange', 'timepicker', 'login',
|
||||
// Click aliases
|
||||
'click+confirm', 'click_download', 'click_dropdown', 'click_checkbox',
|
||||
'clickFirstRow', 'clickInModal', 'delete', 'download',
|
||||
// Navigation aliases
|
||||
'navigateBack', 'goBack', 'navigation', 'directNavigation',
|
||||
'navigateViaMenuClick', 'refresh',
|
||||
// Wait aliases
|
||||
'waitForModal', 'waitForNavigation', 'waitForTable',
|
||||
'waitForDialogReady', 'wait_dialog_ready', 'wait_for_dialog_ready',
|
||||
// Verify aliases
|
||||
'verify', 'verify_elements', 'verify_table_data', 'verify_table_structure',
|
||||
'verify_field', 'verify_checkbox', 'verify_action_buttons', 'verify_pagination',
|
||||
'verify_summary', 'verify_totals', 'verify_search_result', 'verify_row_count',
|
||||
'verify_vendor_info', 'verify_detail_info', 'verify_data_update',
|
||||
'verify_transactions_update', 'verify_transaction_table', 'verify_calculated_value',
|
||||
'verifyButtonExists', 'verifyUrl', 'verifyNoErrorPage',
|
||||
// Select aliases
|
||||
'combobox', 'select_option', 'select_or_click',
|
||||
// Modal aliases
|
||||
'closeModal', 'modalClose', 'openModal', 'checkModalOpen',
|
||||
'fillInModal', 'selectInModal',
|
||||
// Other aliases
|
||||
'confirm_dialog', 'store', 'getCurrentUrl', 'clearSearch',
|
||||
'keypress', 'press', 'pressKey', 'generateTimestamp', 'random',
|
||||
'setupDownloadListener', 'saveDownloadedFile', 'verifyDownload',
|
||||
'verifyDownloadedFile', 'screenshot', 'capture',
|
||||
'toggle_switch', 'radio', 'resize', 'log', 'manualVerification',
|
||||
'composite', 'hierarchy', 'permission', 'ifStillFailed', 'tryAlternativeUrls',
|
||||
'element', 'elementExists', 'tableExists', 'tabsExist',
|
||||
'error_message', 'warning', 'url', 'URL_STABILITY', 'checkFor404',
|
||||
'expectResponse', 'assertResponse', 'apiResponse',
|
||||
'findRow', 'scroll',
|
||||
// Extended aliases
|
||||
'a11y_audit', 'accessibility', 'perf_measure', 'performance',
|
||||
'boundary_fill', 'rapid', 'save_context', 'context_save',
|
||||
'load_context', 'context_load', 'noop',
|
||||
]);
|
||||
|
||||
const DEPRECATED_PATTERNS = [
|
||||
{ pattern: 'window.__API_LOGS__', fix: 'window.__E2E__.getApiLogs().logs', severity: 'error' },
|
||||
{ pattern: 'window.__API_ERRORS__', fix: 'window.__E2E__.getApiLogs().errors', severity: 'error' },
|
||||
];
|
||||
|
||||
let totalIssues = 0;
|
||||
let errorCount = 0;
|
||||
let warnCount = 0;
|
||||
|
||||
console.log(C.bold('\n=== 시나리오 정합성 검증 ===\n'));
|
||||
|
||||
for (const f of files) {
|
||||
const fPath = path.join(SCENARIOS_DIR, f);
|
||||
const issues = [];
|
||||
|
||||
try {
|
||||
const raw = fs.readFileSync(fPath, 'utf-8');
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(raw);
|
||||
} catch (parseErr) {
|
||||
issues.push({ severity: 'error', msg: `JSON 파싱 실패: ${parseErr.message}` });
|
||||
errorCount++;
|
||||
totalIssues++;
|
||||
console.log(`${C.red('✘')} ${f}`);
|
||||
issues.forEach(i => console.log(` ${i.severity === 'error' ? C.red('ERROR') : C.yellow('WARN')}: ${i.msg}`));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Required fields
|
||||
if (!data.id) issues.push({ severity: 'warn', msg: 'id 필드 누락' });
|
||||
if (!data.name) issues.push({ severity: 'warn', msg: 'name 필드 누락' });
|
||||
if (!data.steps || !Array.isArray(data.steps)) {
|
||||
issues.push({ severity: 'error', msg: 'steps 배열 누락 또는 잘못된 형식' });
|
||||
}
|
||||
if (!data.menuNavigation) issues.push({ severity: 'warn', msg: 'menuNavigation 누락' });
|
||||
|
||||
// Step validation
|
||||
if (data.steps && Array.isArray(data.steps)) {
|
||||
for (const step of data.steps) {
|
||||
const action = step.action || step.type;
|
||||
if (action && !VALID_ACTIONS.has(action) && !VALID_ALIASES.has(action)) {
|
||||
issues.push({ severity: 'warn', msg: `스텝 #${step.id}: 알 수 없는 액션 "${action}"` });
|
||||
}
|
||||
// Duplicate step IDs
|
||||
const idCount = data.steps.filter(s => s.id === step.id).length;
|
||||
if (idCount > 1 && step === data.steps.find(s => s.id === step.id)) {
|
||||
issues.push({ severity: 'error', msg: `스텝 ID ${step.id} 중복 (${idCount}회)` });
|
||||
}
|
||||
}
|
||||
|
||||
// Step ID sequencing
|
||||
const ids = data.steps.map(s => s.id).filter(id => typeof id === 'number');
|
||||
for (let i = 1; i < ids.length; i++) {
|
||||
if (ids[i] <= ids[i - 1]) {
|
||||
issues.push({ severity: 'warn', msg: `스텝 ID 순서 비정상: #${ids[i-1]} → #${ids[i]}` });
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deprecated pattern check (in raw text)
|
||||
for (const dp of DEPRECATED_PATTERNS) {
|
||||
if (raw.includes(dp.pattern)) {
|
||||
issues.push({ severity: dp.severity, msg: `deprecated 패턴: "${dp.pattern}" → "${dp.fix}"` });
|
||||
}
|
||||
}
|
||||
|
||||
// Enabled check
|
||||
if (data.enabled === false) {
|
||||
issues.push({ severity: 'warn', msg: 'enabled: false (비활성 시나리오)' });
|
||||
}
|
||||
|
||||
// menuNavigation consistency
|
||||
if (data.menuNavigation && data.steps) {
|
||||
const menuStep = data.steps.find(s => s.action === 'menu_navigate');
|
||||
if (menuStep) {
|
||||
if (menuStep.level1 !== data.menuNavigation.level1 || menuStep.level2 !== data.menuNavigation.level2) {
|
||||
issues.push({ severity: 'warn', msg: `menuNavigation과 menu_navigate 스텝 불일치: ${data.menuNavigation.level1}>${data.menuNavigation.level2} vs ${menuStep.level1}>${menuStep.level2}` });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (readErr) {
|
||||
issues.push({ severity: 'error', msg: `파일 읽기 실패: ${readErr.message}` });
|
||||
}
|
||||
|
||||
// Tally
|
||||
const errors = issues.filter(i => i.severity === 'error');
|
||||
const warns = issues.filter(i => i.severity === 'warn');
|
||||
errorCount += errors.length;
|
||||
warnCount += warns.length;
|
||||
totalIssues += issues.length;
|
||||
|
||||
if (issues.length > 0) {
|
||||
const icon = errors.length > 0 ? C.red('✘') : C.yellow('△');
|
||||
console.log(`${icon} ${f} (${errors.length} errors, ${warns.length} warns)`);
|
||||
issues.forEach(i => {
|
||||
const prefix = i.severity === 'error' ? C.red(' ERROR') : C.yellow(' WARN');
|
||||
console.log(`${prefix}: ${i.msg}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Summary
|
||||
const okCount = files.length - (new Set([...files.filter(f => {
|
||||
try {
|
||||
const raw = fs.readFileSync(path.join(SCENARIOS_DIR, f), 'utf-8');
|
||||
return DEPRECATED_PATTERNS.some(dp => raw.includes(dp.pattern));
|
||||
} catch { return false; }
|
||||
})])).size;
|
||||
|
||||
console.log(`\n${C.bold('─── 검증 결과 ───')}`);
|
||||
console.log(` 전체 시나리오: ${C.bold(files.length)}개`);
|
||||
console.log(` 이슈: ${totalIssues > 0 ? C.yellow(totalIssues) : C.green('0')}건 (에러: ${errorCount}, 경고: ${warnCount})`);
|
||||
console.log(` ${totalIssues === 0 ? C.green('✅ 모든 시나리오 정합성 통과') : C.yellow(`⚠️ ${totalIssues}건의 이슈 발견`)}\n`);
|
||||
|
||||
return { total: files.length, errors: errorCount, warns: warnCount };
|
||||
}
|
||||
|
||||
// ─── Executor Injection ─────────────────────────────────────
|
||||
|
||||
const executorCode = fs.readFileSync(EXECUTOR_PATH, 'utf-8');
|
||||
@@ -591,8 +919,8 @@ async function verifyPageHealth(page, timeout = 8000) {
|
||||
'table, [class*="content"], main, [role="main"], [class*="page"], [class*="list"]'
|
||||
);
|
||||
|
||||
// Console errors (if captured by step-executor ApiMonitor)
|
||||
const apiErrors = (window.__API_ERRORS__ || []).filter(e =>
|
||||
// API errors (from step-executor ApiMonitor)
|
||||
const apiErrors = (window.__E2E__ ? window.__E2E__.getApiLogs().errors : []).filter(e =>
|
||||
e.status >= 400 || e.error
|
||||
);
|
||||
|
||||
@@ -730,8 +1058,9 @@ async function diagnoseFail(page, result, scenarioId) {
|
||||
const errorBoundary = errorPatterns.find(p => bodyText.includes(p));
|
||||
|
||||
// API errors from step-executor monitor
|
||||
const apiLogs = window.__API_LOGS__ || [];
|
||||
const apiErrors = (window.__API_ERRORS__ || []).slice(0, 10).map(e => ({
|
||||
const apiData = window.__E2E__ ? window.__E2E__.getApiLogs() : { logs: [], errors: [] };
|
||||
const apiLogs = apiData.logs || [];
|
||||
const apiErrors = (apiData.errors || []).slice(0, 10).map(e => ({
|
||||
url: (e.url || '').substring(0, 120),
|
||||
status: e.status,
|
||||
method: e.method,
|
||||
@@ -1206,12 +1535,104 @@ function saveReport(result, timestamp) {
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Stability Tracking ─────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Read the most recent previous summary report to detect trends.
|
||||
* Returns: { prevPassed, prevFailed, prevTotal, prevTimestamp, scenarioResults: Map<id, 'pass'|'fail'> }
|
||||
*/
|
||||
function getPreviousRunResults() {
|
||||
try {
|
||||
const summaryFiles = fs.readdirSync(RESULTS_DIR)
|
||||
.filter(f => f.startsWith('E2E_FULL_TEST_SUMMARY_') && f.endsWith('.md'))
|
||||
.sort()
|
||||
.reverse();
|
||||
|
||||
if (summaryFiles.length === 0) return null;
|
||||
|
||||
const content = fs.readFileSync(path.join(RESULTS_DIR, summaryFiles[0]), 'utf-8');
|
||||
const scenarioResults = new Map();
|
||||
|
||||
// Parse scenario results from table
|
||||
const tableLines = content.split('\n').filter(l => l.startsWith('| ') && /\| [✅❌]/.test(l));
|
||||
tableLines.forEach(line => {
|
||||
const cols = line.split('|').map(c => c.trim()).filter(Boolean);
|
||||
if (cols.length >= 3) {
|
||||
// cols: [#, 시나리오명, 결과, ...]
|
||||
const name = cols[1];
|
||||
const status = cols[2].includes('✅') ? 'pass' : 'fail';
|
||||
scenarioResults.set(name, status);
|
||||
}
|
||||
});
|
||||
|
||||
// Parse summary stats
|
||||
const totalMatch = content.match(/전체 시나리오.*?(\d+)개.*?성공.*?(\d+)개.*?실패.*?(\d+)개/);
|
||||
const timeMatch = content.match(/실행 시간.*?:\s*(.+)/);
|
||||
|
||||
return {
|
||||
prevTotal: totalMatch ? parseInt(totalMatch[1]) : 0,
|
||||
prevPassed: totalMatch ? parseInt(totalMatch[2]) : 0,
|
||||
prevFailed: totalMatch ? parseInt(totalMatch[3]) : 0,
|
||||
prevTimestamp: timeMatch ? timeMatch[1].trim() : 'unknown',
|
||||
scenarioResults,
|
||||
fileName: summaryFiles[0],
|
||||
};
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect flaky tests by checking recent OK/Fail report files.
|
||||
* A test is flaky if it has both OK- and Fail- reports in the last N runs.
|
||||
* Returns: Map<scenarioId, { passCount, failCount }>
|
||||
*/
|
||||
function detectFlakyTests(lookbackDays = 3) {
|
||||
const flakyMap = new Map();
|
||||
const cutoff = Date.now() - (lookbackDays * 24 * 60 * 60 * 1000);
|
||||
|
||||
try {
|
||||
// Check success reports
|
||||
const okFiles = fs.readdirSync(SUCCESS_DIR).filter(f => f.startsWith('OK-') && f.endsWith('.md'));
|
||||
okFiles.forEach(f => {
|
||||
const stat = fs.statSync(path.join(SUCCESS_DIR, f));
|
||||
if (stat.mtimeMs < cutoff) return;
|
||||
const id = f.replace(/^OK-/, '').replace(/_\d{4}-\d{2}-\d{2}.*$/, '');
|
||||
if (!flakyMap.has(id)) flakyMap.set(id, { passCount: 0, failCount: 0 });
|
||||
flakyMap.get(id).passCount++;
|
||||
});
|
||||
|
||||
// Check fail reports
|
||||
const failFiles = fs.readdirSync(RESULTS_DIR).filter(f => f.startsWith('Fail-') && f.endsWith('.md'));
|
||||
failFiles.forEach(f => {
|
||||
const stat = fs.statSync(path.join(RESULTS_DIR, f));
|
||||
if (stat.mtimeMs < cutoff) return;
|
||||
const id = f.replace(/^Fail-/, '').replace(/_\d{4}-\d{2}-\d{2}.*$/, '');
|
||||
if (!flakyMap.has(id)) flakyMap.set(id, { passCount: 0, failCount: 0 });
|
||||
flakyMap.get(id).failCount++;
|
||||
});
|
||||
} catch (e) { /* ignore */ }
|
||||
|
||||
// Filter to only include tests that have BOTH passes and failures
|
||||
const flaky = new Map();
|
||||
for (const [id, counts] of flakyMap) {
|
||||
if (counts.passCount > 0 && counts.failCount > 0) {
|
||||
flaky.set(id, counts);
|
||||
}
|
||||
}
|
||||
return flaky;
|
||||
}
|
||||
|
||||
// ─── Summary Report ─────────────────────────────────────────
|
||||
|
||||
function generateSummaryReport(allResults, totalTime, timestamp) {
|
||||
const passed = allResults.filter((r) => !r.error && r.failed === 0).length;
|
||||
const failed = allResults.length - passed;
|
||||
|
||||
// Get previous run for trend analysis
|
||||
const prevRun = getPreviousRunResults();
|
||||
const flakyTests = detectFlakyTests();
|
||||
|
||||
// Categorize results
|
||||
const categories = {};
|
||||
allResults.forEach((r) => {
|
||||
@@ -1220,11 +1641,19 @@ function generateSummaryReport(allResults, totalTime, timestamp) {
|
||||
categories[cat].push(r);
|
||||
});
|
||||
|
||||
// Trend indicators
|
||||
const trendIcon = prevRun
|
||||
? (passed > prevRun.prevPassed ? '📈' : passed < prevRun.prevPassed ? '📉' : '➡️')
|
||||
: '';
|
||||
const trendText = prevRun
|
||||
? ` ${trendIcon} (이전: ${prevRun.prevPassed}/${prevRun.prevTotal} 성공)`
|
||||
: '';
|
||||
|
||||
let md = `# E2E 전체 테스트 결과 요약
|
||||
|
||||
**실행 시간**: ${timestamp}
|
||||
**총 소요 시간**: ${(totalTime / 1000 / 60).toFixed(1)}분
|
||||
**전체 시나리오**: ${allResults.length}개 | **성공**: ${passed}개 | **실패**: ${failed}개
|
||||
**전체 시나리오**: ${allResults.length}개 | **성공**: ${passed}개 | **실패**: ${failed}개${trendText}
|
||||
|
||||
## 카테고리별 요약
|
||||
| 카테고리 | 시나리오 수 | 성공 | 실패 | 성공률 |
|
||||
@@ -1334,17 +1763,85 @@ function generateSummaryReport(allResults, totalTime, timestamp) {
|
||||
});
|
||||
}
|
||||
|
||||
// Flaky test detection section
|
||||
if (flakyTests.size > 0) {
|
||||
md += `\n## ⚠️ 불안정 테스트 (Flaky Tests)\n`;
|
||||
md += `최근 3일간 성공과 실패가 모두 발생한 시나리오:\n\n`;
|
||||
md += `| 시나리오 | 성공 횟수 | 실패 횟수 | 안정성 |\n`;
|
||||
md += `|---------|----------|----------|--------|\n`;
|
||||
const sorted = [...flakyTests.entries()].sort((a, b) => b[1].failCount - a[1].failCount);
|
||||
sorted.forEach(([id, counts]) => {
|
||||
const total = counts.passCount + counts.failCount;
|
||||
const stability = Math.round((counts.passCount / total) * 100);
|
||||
const grade = stability >= 80 ? '🟡' : stability >= 50 ? '🟠' : '🔴';
|
||||
md += `| ${id} | ${counts.passCount} | ${counts.failCount} | ${grade} ${stability}% |\n`;
|
||||
});
|
||||
}
|
||||
|
||||
// Trend analysis section
|
||||
if (prevRun) {
|
||||
md += `\n## 📊 트렌드 분석\n`;
|
||||
md += `| 항목 | 이전 실행 | 현재 실행 | 변화 |\n`;
|
||||
md += `|------|---------|---------|------|\n`;
|
||||
const passedDiff = passed - prevRun.prevPassed;
|
||||
const failedDiff = failed - prevRun.prevFailed;
|
||||
const passedIcon = passedDiff > 0 ? `📈 +${passedDiff}` : passedDiff < 0 ? `📉 ${passedDiff}` : '➡️ 동일';
|
||||
const failedIcon = failedDiff < 0 ? `📈 ${failedDiff}` : failedDiff > 0 ? `📉 +${failedDiff}` : '➡️ 동일';
|
||||
md += `| 전체 | ${prevRun.prevTotal} | ${allResults.length} | ${allResults.length - prevRun.prevTotal >= 0 ? '+' : ''}${allResults.length - prevRun.prevTotal} |\n`;
|
||||
md += `| 성공 | ${prevRun.prevPassed} | ${passed} | ${passedIcon} |\n`;
|
||||
md += `| 실패 | ${prevRun.prevFailed} | ${failed} | ${failedIcon} |\n`;
|
||||
md += `\n이전 실행: ${prevRun.prevTimestamp}\n`;
|
||||
|
||||
// Newly failed / newly passed scenarios
|
||||
if (prevRun.scenarioResults.size > 0) {
|
||||
const newlyFailed = [];
|
||||
const newlyPassed = [];
|
||||
allResults.forEach(r => {
|
||||
const currentStatus = (!r.error && r.failed === 0) ? 'pass' : 'fail';
|
||||
const prevStatus = prevRun.scenarioResults.get(r.name);
|
||||
if (prevStatus === 'pass' && currentStatus === 'fail') newlyFailed.push(r.name);
|
||||
if (prevStatus === 'fail' && currentStatus === 'pass') newlyPassed.push(r.name);
|
||||
});
|
||||
if (newlyFailed.length > 0) {
|
||||
md += `\n### 🔴 새로 실패한 시나리오 (${newlyFailed.length}개)\n`;
|
||||
newlyFailed.forEach(n => md += `- ${n}\n`);
|
||||
}
|
||||
if (newlyPassed.length > 0) {
|
||||
md += `\n### 🟢 새로 통과한 시나리오 (${newlyPassed.length}개)\n`;
|
||||
newlyPassed.forEach(n => md += `- ${n}\n`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return md;
|
||||
}
|
||||
|
||||
// ─── Main ───────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
// --groups: 그룹 목록만 출력하고 종료
|
||||
if (LIST_GROUPS) {
|
||||
listGroups();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// --validate: 시나리오 정합성 검증만 수행하고 종료
|
||||
if (VALIDATE_ONLY) {
|
||||
let files = fs.readdirSync(SCENARIOS_DIR)
|
||||
.filter(f => f.endsWith('.json') && !f.startsWith('_'))
|
||||
.sort();
|
||||
if (FILTER) files = files.filter(f => f.includes(FILTER));
|
||||
if (GROUP) files = filterByGroup(files, GROUP);
|
||||
const result = validateScenarios(files);
|
||||
process.exit(result.errors > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
console.log(C.bold('\n=== E2E 전체 테스트 러너 ==='));
|
||||
console.log(`서버: ${BASE_URL}`);
|
||||
console.log(`모드: ${HEADLESS ? 'headless' : 'headed'}`);
|
||||
if (WORKFLOW_ONLY) console.log(`카테고리: workflow only`);
|
||||
if (FILTER) console.log(`필터: ${FILTER}`);
|
||||
if (GROUP) console.log(`그룹: ${GROUP}`);
|
||||
if (EXCLUDE) console.log(`제외: ${EXCLUDE}`);
|
||||
if (SKIP_PASSED) console.log(`스킵: 이미 성공한 시나리오 건너뛰기 (--skip-passed)`);
|
||||
if (ITERATE) console.log(`반복 모드: 실패 시나리오 자동 재실행 (최대 ${MAX_ITERATIONS}회)`);
|
||||
@@ -1379,6 +1876,9 @@ async function main() {
|
||||
if (EXCLUDE) {
|
||||
scenarioFiles = scenarioFiles.filter((f) => !f.includes(EXCLUDE));
|
||||
}
|
||||
if (GROUP) {
|
||||
scenarioFiles = filterByGroup(scenarioFiles, GROUP);
|
||||
}
|
||||
|
||||
// --skip-passed: 이미 성공한 시나리오 건너뛰기
|
||||
let skippedCount = 0;
|
||||
|
||||
@@ -361,6 +361,8 @@
|
||||
waitForModal: 'wait_for_modal',
|
||||
waitForNavigation: 'wait_for_navigation',
|
||||
waitForTable: 'wait_for_table',
|
||||
waitForDialogReady: 'wait_for_dialog_ready',
|
||||
wait_dialog_ready: 'wait_for_dialog_ready',
|
||||
// Verify variants
|
||||
verify_elements: 'verify_element',
|
||||
verify_table_data: 'verify_table',
|
||||
@@ -962,14 +964,64 @@
|
||||
|
||||
async wait_for_modal(action, ctx) {
|
||||
const timeout = action.timeout || 5000;
|
||||
const waitForReady = action.waitForReady !== false; // default: also check form readiness
|
||||
const t0 = now();
|
||||
while (now() - t0 < timeout) {
|
||||
if (ModalGuard.check().open) return pass('Modal appeared');
|
||||
const { open, element } = ModalGuard.check();
|
||||
if (open && element) {
|
||||
if (!waitForReady) return pass('Modal appeared');
|
||||
// Also verify modal is interactive (has rendered form fields or buttons)
|
||||
const hasInteractive = element.querySelector(
|
||||
'input:not([type="hidden"]), textarea, select, button:not([class*="close"]), [role="button"]'
|
||||
);
|
||||
if (hasInteractive) {
|
||||
return pass(`Modal ready (interactive elements found)`);
|
||||
}
|
||||
// Modal appeared but not yet interactive - keep waiting
|
||||
}
|
||||
await sleep(200);
|
||||
}
|
||||
// Final check: modal appeared but may not have interactive elements (read-only modals)
|
||||
const { open, element } = ModalGuard.check();
|
||||
if (open) {
|
||||
return pass('Modal appeared (no interactive elements detected)');
|
||||
}
|
||||
return fail('Timeout waiting for modal');
|
||||
},
|
||||
|
||||
/** Enhanced wait: waits for dialog AND verifies form fields are ready */
|
||||
async wait_for_dialog_ready(action, ctx) {
|
||||
const timeout = action.timeout || 8000;
|
||||
const t0 = now();
|
||||
while (now() - t0 < timeout) {
|
||||
const { open, element } = ModalGuard.check();
|
||||
if (open && element) {
|
||||
// Check that form fields are not just present but also visible and enabled
|
||||
const inputs = element.querySelectorAll('input:not([type="hidden"]), textarea, select');
|
||||
const visibleInputs = Array.from(inputs).filter(el => {
|
||||
const rect = el.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0 && !el.disabled;
|
||||
});
|
||||
if (visibleInputs.length > 0) {
|
||||
return pass(`Dialog ready: ${visibleInputs.length} input fields visible`);
|
||||
}
|
||||
// Also check for buttons (confirmation dialogs without inputs)
|
||||
const buttons = element.querySelectorAll('button:not([class*="close"])');
|
||||
const visibleButtons = Array.from(buttons).filter(el => {
|
||||
const rect = el.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
});
|
||||
if (visibleButtons.length > 0) {
|
||||
return pass(`Dialog ready: ${visibleButtons.length} buttons visible`);
|
||||
}
|
||||
}
|
||||
await sleep(250);
|
||||
}
|
||||
const { open } = ModalGuard.check();
|
||||
if (open) return warn('Dialog appeared but no interactive elements found');
|
||||
return fail('Timeout waiting for dialog');
|
||||
},
|
||||
|
||||
async wait_for_navigation(action, ctx) {
|
||||
await sleep(2000);
|
||||
const v = action.expected || action.verification;
|
||||
@@ -1841,22 +1893,76 @@
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
lastResult = await handler(action, ctx);
|
||||
if (lastResult.status === 'pass' || lastResult.status === 'navigation' || lastResult.status === 'native_required') {
|
||||
if (attempt > 0) {
|
||||
lastResult.details = `[retry ${attempt}] ${lastResult.details}`;
|
||||
}
|
||||
return lastResult;
|
||||
}
|
||||
if (attempt < maxRetries) {
|
||||
// Pre-retry actions
|
||||
if (lastResult.details?.includes('not found') || lastResult.details?.includes('not visible')) {
|
||||
await sleep(delayMs);
|
||||
// Try closing overlays
|
||||
// Progressive delay: 500ms → 1000ms → 1500ms
|
||||
const progressiveDelay = delayMs * (attempt + 1);
|
||||
|
||||
// Pre-retry actions based on failure type
|
||||
const errorMsg = lastResult.details || '';
|
||||
if (errorMsg.includes('not found') || errorMsg.includes('not visible')) {
|
||||
// Element not found: try closing overlays and scrolling
|
||||
if (ModalGuard.check().open) await ModalGuard.close();
|
||||
await sleep(progressiveDelay);
|
||||
// Try scrolling the target into view on retry
|
||||
if (action.target) {
|
||||
try {
|
||||
const el = document.querySelector(action.target);
|
||||
if (el) el.scrollIntoView({ block: 'center', behavior: 'instant' });
|
||||
} catch (_) { /* invalid selector, ignore */ }
|
||||
}
|
||||
} else if (errorMsg.includes('not clickable') || errorMsg.includes('intercepted')) {
|
||||
// Click intercepted: close overlays and wait
|
||||
if (ModalGuard.check().open) await ModalGuard.close();
|
||||
await sleep(progressiveDelay);
|
||||
} else {
|
||||
await sleep(delayMs);
|
||||
await sleep(progressiveDelay);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Enrich failure message with DOM context
|
||||
if (lastResult.status === 'fail' && action.target) {
|
||||
const context = getDomContext(action.target);
|
||||
if (context) {
|
||||
lastResult.details = `${lastResult.details} [context: ${context}]`;
|
||||
}
|
||||
}
|
||||
return lastResult;
|
||||
}
|
||||
|
||||
/** Get brief DOM context around a failed selector for debugging */
|
||||
function getDomContext(selector) {
|
||||
try {
|
||||
// Check if any similar elements exist
|
||||
const allButtons = document.querySelectorAll('button');
|
||||
const allInputs = document.querySelectorAll('input:not([type="hidden"])');
|
||||
const modalOpen = ModalGuard.check().open;
|
||||
|
||||
const parts = [];
|
||||
if (modalOpen) parts.push('modal:open');
|
||||
parts.push(`btn:${allButtons.length}`);
|
||||
parts.push(`input:${allInputs.length}`);
|
||||
|
||||
// If selector contains Korean text, check if similar text exists on page
|
||||
if (/[가-힣]/.test(selector)) {
|
||||
const bodyText = document.body.innerText || '';
|
||||
const cleanSel = selector.replace(/[^가-힣\s]/g, '').trim();
|
||||
if (cleanSel && bodyText.includes(cleanSel)) {
|
||||
parts.push('text:found-on-page');
|
||||
} else {
|
||||
parts.push('text:not-on-page');
|
||||
}
|
||||
}
|
||||
return parts.join(',');
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Step Timeout ───────────────────────────────────────
|
||||
|
||||
const STEP_TIMEOUT_MS = 60000; // 60초: 개별 스텝 최대 대기 시간
|
||||
|
||||
Reference in New Issue
Block a user