diff --git a/e2e/runner/run-all.js b/e2e/runner/run-all.js index e8dbb0a..9dbf24d 100644 --- a/e2e/runner/run-all.js +++ b/e2e/runner/run-all.js @@ -13,6 +13,9 @@ * node e2e/runner/run-all.js --headless # headless * node e2e/runner/run-all.js --exclude sales # 파일명에 "sales" 포함된 것 제외 * node e2e/runner/run-all.js --skip-passed # 이미 성공한 시나리오 건너뛰기 + * node e2e/runner/run-all.js --iterate # 실패 시나리오 자동 재실행 (최대 3회) + * node e2e/runner/run-all.js --iterate 5 # 실패 시나리오 자동 재실행 (최대 5회) + * node e2e/runner/run-all.js --stage # 카테고리별 단계 실행 (accessibility → edge → perf → workflow → functional) */ const fs = require('fs'); @@ -46,6 +49,16 @@ const EXCLUDE = (() => { return idx >= 0 && args[idx + 1] ? args[idx + 1] : null; })(); const SKIP_PASSED = args.includes('--skip-passed'); +const ITERATE = args.includes('--iterate'); +const MAX_ITERATIONS = (() => { + const idx = args.indexOf('--iterate'); + if (idx >= 0 && args[idx + 1] && !args[idx + 1].startsWith('--')) { + const n = parseInt(args[idx + 1], 10); + return isNaN(n) ? 3 : Math.min(Math.max(n, 1), 10); + } + return 3; +})(); +const STAGE_MODE = args.includes('--stage'); // ─── Helpers ──────────────────────────────────────────────── @@ -1270,6 +1283,8 @@ async function main() { if (FILTER) console.log(`필터: ${FILTER}`); if (EXCLUDE) console.log(`제외: ${EXCLUDE}`); if (SKIP_PASSED) console.log(`스킵: 이미 성공한 시나리오 건너뛰기 (--skip-passed)`); + if (ITERATE) console.log(`반복 모드: 실패 시나리오 자동 재실행 (최대 ${MAX_ITERATIONS}회)`); + if (STAGE_MODE) console.log(`단계 모드: 카테고리별 순차 실행`); console.log(''); // Ensure directories @@ -1371,24 +1386,40 @@ async function main() { process.exit(1); } - // Run scenarios - const allResults = []; - const startTime = Date.now(); + // ─── Helper: Run a list of scenario files and return results ──── + async function runScenarioList(files, label, counter) { + const results = []; + for (let i = 0; i < files.length; i++) { + const file = files[i]; + const scenarioPath = path.join(SCENARIOS_DIR, file); + counter.current++; + const num = `(${counter.current}/${counter.total})`; - for (let i = 0; i < scenarioFiles.length; i++) { - const file = scenarioFiles[i]; - const scenarioPath = path.join(SCENARIOS_DIR, file); - const num = `(${i + 1}/${totalScenarios})`; + process.stdout.write(`${C.dim(num)} ${label ? `[${label}] ` : ''}${file.replace('.json', '')} ... `); - process.stdout.write(`${C.dim(num)} ${file.replace('.json', '')} ... `); - - let result; - const timeout = getScenarioTimeout(file); - try { - // Wrap with timeout - result = await Promise.race([ - runScenario(page, scenarioPath), - sleep(timeout).then(() => ({ + let result; + const timeout = getScenarioTimeout(file); + try { + result = await Promise.race([ + runScenario(page, scenarioPath), + sleep(timeout).then(() => ({ + id: file.replace('.json', ''), + name: file.replace('.json', ''), + steps: [], + passed: 0, + failed: 0, + warned: 0, + totalSteps: 0, + apiSummary: null, + error: `Timeout (>${timeout / 1000}s)`, + stoppedReason: 'timeout', + currentUrl: '', + startTime: Date.now() - timeout, + endTime: Date.now(), + })), + ]); + } catch (scenarioErr) { + result = { id: file.replace('.json', ''), name: file.replace('.json', ''), steps: [], @@ -1397,66 +1428,213 @@ async function main() { warned: 0, totalSteps: 0, apiSummary: null, - error: `Timeout (>${timeout / 1000}s)`, - stoppedReason: 'timeout', + error: scenarioErr.message, + stoppedReason: 'exception', currentUrl: '', - startTime: Date.now() - timeout, + startTime: Date.now(), endTime: Date.now(), - })), - ]); - } catch (scenarioErr) { - result = { - id: file.replace('.json', ''), - name: file.replace('.json', ''), - steps: [], - passed: 0, - failed: 0, - warned: 0, - totalSteps: 0, - apiSummary: null, - error: scenarioErr.message, - stoppedReason: 'exception', - currentUrl: '', - startTime: Date.now(), - endTime: Date.now(), - }; + }; + } + + results.push(result); + + // Save report + const ts = getTimestamp(); + saveReport(result, ts); + + // Console output + const hasFail = result.failed > 0 || result.error; + const duration = ((result.endTime - result.startTime) / 1000).toFixed(1); + if (hasFail) { + console.log(`${C.red('FAIL')} ${C.dim(`(${result.passed}/${result.totalSteps} passed, ${duration}s)`)}`); + } else { + console.log(`${C.green('PASS')} ${C.dim(`(${result.passed}/${result.totalSteps}, ${duration}s)`)}`); + } + + // After login scenario or if logged out, re-login + if (file === 'login.json' || page.url().includes('/login')) { + try { + await page.goto(`${BASE_URL}/ko/login`, { waitUntil: 'domcontentloaded', timeout: 10000 }); + await sleep(500); + await page.fill('#userId', AUTH.username); + await page.fill('#password', AUTH.password); + await page.click("button[type='submit']"); + await sleep(3000); + } catch (reloginErr) { + // Continue anyway + } + } + } + return results; + } + + // ─── Stage Mode: Category-by-category execution ──────────── + const STAGE_ORDER = ['accessibility', 'edge-case', 'performance', 'workflow', 'functional']; + const STAGE_NAMES = { + 'accessibility': '접근성 검사', + 'edge-case': '엣지 케이스', + 'performance': '성능 테스트', + 'workflow': '비즈니스 워크플로우', + 'functional': '기능 테스트', + }; + + // Run scenarios + const allResults = []; + const iterationHistory = []; // Track iteration progress + const startTime = Date.now(); + + if (STAGE_MODE) { + // ─── STAGE MODE: Run by category ───────────────────────── + const categorized = {}; + for (const file of scenarioFiles) { + const cat = getScenarioCategory(file); + if (!categorized[cat]) categorized[cat] = []; + categorized[cat].push(file); } - allResults.push(result); + let stageNum = 0; + const counter = { current: 0, total: totalScenarios }; - // Save report - const ts = getTimestamp(); - saveReport(result, ts); + for (const cat of STAGE_ORDER) { + const files = categorized[cat]; + if (!files || files.length === 0) continue; - // Console output - const hasFail = result.failed > 0 || result.error; - const duration = ((result.endTime - result.startTime) / 1000).toFixed(1); - if (hasFail) { - console.log(`${C.red('FAIL')} ${C.dim(`(${result.passed}/${result.totalSteps} passed, ${duration}s)`)}`); - } else { - console.log(`${C.green('PASS')} ${C.dim(`(${result.passed}/${result.totalSteps}, ${duration}s)`)}`); + stageNum++; + const catName = STAGE_NAMES[cat] || cat; + console.log(C.bold(`\n── Stage ${stageNum}: ${catName} (${files.length}개) ──`)); + + const stageResults = await runScenarioList(files, catName, counter); + allResults.push(...stageResults); + + // Stage summary + const stagePassed = stageResults.filter(r => !r.error && r.failed === 0).length; + const stageFailed = stageResults.length - stagePassed; + const stageRate = stageResults.length > 0 ? Math.round((stagePassed / stageResults.length) * 100) : 0; + + console.log(C.cyan(`\n ▸ ${catName} 결과: ${stagePassed}/${stageResults.length} PASS (${stageRate}%)`)); + if (stageFailed > 0) { + const failNames = stageResults.filter(r => r.failed > 0 || r.error).map(r => r.id); + console.log(C.red(` ▸ 실패: ${failNames.join(', ')}`)); + } } - // After login scenario or if logged out, re-login - if (file === 'login.json' || page.url().includes('/login')) { - try { - await page.goto(`${BASE_URL}/ko/login`, { waitUntil: 'domcontentloaded', timeout: 10000 }); - await sleep(500); - await page.fill('#userId', AUTH.username); - await page.fill('#password', AUTH.password); - await page.click("button[type='submit']"); - await sleep(3000); - } catch (reloginErr) { - // Continue anyway + } else { + // ─── NORMAL MODE: Run all sequentially ─────────────────── + const counter = { current: 0, total: totalScenarios }; + const results = await runScenarioList(scenarioFiles, '', counter); + allResults.push(...results); + } + + // ─── Iteration 0 (initial run) summary ─────────────────────── + const initialPassed = allResults.filter(r => !r.error && r.failed === 0).length; + const initialFailed = allResults.length - initialPassed; + + iterationHistory.push({ + iteration: 0, + label: '초기 실행', + total: allResults.length, + passed: initialPassed, + failed: initialFailed, + rate: allResults.length > 0 ? Math.round((initialPassed / allResults.length) * 100) : 0, + fixed: [], + stillFailing: allResults.filter(r => r.failed > 0 || r.error).map(r => r.id), + }); + + // ─── ITERATE MODE: Re-run failed scenarios ───────────────── + if (ITERATE && initialFailed > 0) { + for (let iteration = 1; iteration <= MAX_ITERATIONS; iteration++) { + const failedIds = allResults.filter(r => r.failed > 0 || r.error).map(r => r.id); + if (failedIds.length === 0) { + console.log(C.green(C.bold(`\n🎯 반복 ${iteration} 불필요: 모든 시나리오 PASS!`))); + break; + } + + const failedFiles = failedIds.map(id => `${id}.json`).filter(f => fs.existsSync(path.join(SCENARIOS_DIR, f))); + if (failedFiles.length === 0) break; + + console.log(C.bold(C.yellow(`\n═══ 반복 ${iteration}/${MAX_ITERATIONS}: ${failedFiles.length}개 실패 시나리오 재실행 ═══`))); + console.log(C.dim(` 대상: ${failedIds.join(', ')}\n`)); + + const counter = { current: 0, total: failedFiles.length }; + const retryResults = await runScenarioList(failedFiles, `재시도 ${iteration}`, counter); + + // Update allResults with retry results + const fixedInThisIteration = []; + for (const retryResult of retryResults) { + const idx = allResults.findIndex(r => r.id === retryResult.id); + if (idx >= 0) { + const wasFail = allResults[idx].failed > 0 || allResults[idx].error; + const nowPass = retryResult.failed === 0 && !retryResult.error; + if (wasFail && nowPass) { + fixedInThisIteration.push(retryResult.id); + } + allResults[idx] = retryResult; // Replace with latest result + } + } + + // Iteration summary + const iterPassed = allResults.filter(r => !r.error && r.failed === 0).length; + const iterFailed = allResults.length - iterPassed; + const stillFailing = allResults.filter(r => r.failed > 0 || r.error).map(r => r.id); + + iterationHistory.push({ + iteration, + label: `반복 ${iteration}`, + total: allResults.length, + passed: iterPassed, + failed: iterFailed, + rate: allResults.length > 0 ? Math.round((iterPassed / allResults.length) * 100) : 0, + fixed: fixedInThisIteration, + stillFailing, + }); + + console.log(C.cyan(`\n ▸ 반복 ${iteration} 결과: ${iterPassed}/${allResults.length} PASS (${Math.round((iterPassed / allResults.length) * 100)}%)`)); + if (fixedInThisIteration.length > 0) { + console.log(C.green(` ▸ 이번 반복에서 수정됨: ${fixedInThisIteration.join(', ')}`)); + } + if (stillFailing.length > 0) { + console.log(C.red(` ▸ 여전히 실패: ${stillFailing.join(', ')}`)); + } else { + console.log(C.green(C.bold(`\n🎯 모든 시나리오 PASS 달성! (반복 ${iteration}회 만에)`))); + break; } } } const totalTime = Date.now() - startTime; - // Generate summary report + // Generate summary report (with iteration history) const summaryTs = getTimestamp(); - const summaryMd = generateSummaryReport(allResults, totalTime, summaryTs); + let summaryMd = generateSummaryReport(allResults, totalTime, summaryTs); + + // Append iteration history to summary if iterate mode was used + if (ITERATE && iterationHistory.length > 1) { + summaryMd += `\n## 반복 실행 이력\n`; + summaryMd += `| 반복 | 전체 | 성공 | 실패 | 성공률 | 수정됨 | 여전히 실패 |\n`; + summaryMd += `|:----:|:----:|:----:|:----:|:------:|--------|------------|\n`; + for (const h of iterationHistory) { + const fixedStr = h.fixed.length > 0 ? h.fixed.join(', ') : '-'; + const failStr = h.stillFailing.length > 0 ? h.stillFailing.join(', ') : '-'; + summaryMd += `| ${h.label} | ${h.total} | ${h.passed} | ${h.failed} | ${h.rate}% | ${fixedStr} | ${failStr} |\n`; + } + + // Progress bar + summaryMd += `\n### 반복별 추이\n\`\`\`\n`; + for (const h of iterationHistory) { + const barLen = 40; + const filled = Math.round((h.rate / 100) * barLen); + const bar = '█'.repeat(filled) + '░'.repeat(barLen - filled); + summaryMd += `${h.label.padEnd(12)} ${bar} ${h.rate}% (${h.passed}/${h.total})\n`; + } + summaryMd += `\`\`\`\n`; + } + + // Append stage summary if stage mode was used + if (STAGE_MODE) { + summaryMd += `\n## 단계별 실행 순서\n`; + summaryMd += `실행 순서: ${STAGE_ORDER.filter(c => allResults.some(r => getScenarioCategory(r.id) === c)).map(c => STAGE_NAMES[c] || c).join(' → ')}\n`; + } + const summaryPath = path.join(RESULTS_DIR, `E2E_FULL_TEST_SUMMARY_${summaryTs}.md`); fs.writeFileSync(summaryPath, summaryMd, 'utf-8'); @@ -1469,6 +1647,15 @@ async function main() { console.log(C.bold('\n=== 테스트 완료 ===')); console.log(`전체: ${totalScenarios} | ${C.green(`성공: ${passCount}`)} | ${failCount > 0 ? C.red(`실패: ${failCount}`) : '실패: 0'}`); + if (ITERATE && iterationHistory.length > 1) { + const lastIter = iterationHistory[iterationHistory.length - 1]; + const firstIter = iterationHistory[0]; + const improvement = lastIter.passed - firstIter.passed; + if (improvement > 0) { + console.log(C.green(`반복 개선: +${improvement}개 PASS (${firstIter.rate}% → ${lastIter.rate}%)`)); + } + console.log(`반복 횟수: ${iterationHistory.length - 1}회`); + } console.log(`소요 시간: ${(totalTime / 1000 / 60).toFixed(1)}분`); console.log(`요약 리포트: ${summaryPath}`); console.log('');