SamProject/sam-hotfix
4
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: run-all.js에 --iterate 및 --stage 모드 추가

Browse Source 
- --iterate [N]: 실패 시나리오 자동 재실행 (기본 3회, 최대 10회)
  - 초기 실행 후 FAIL 시나리오만 재실행
  - 반복마다 수정된/여전히 실패한 시나리오 추적
  - 요약 리포트에 반복 이력 테이블 + 추이 그래프 포함
  - 모든 시나리오 PASS 달성 시 즉시 종료

- --stage: 카테고리별 단계 실행
  - 실행 순서: 접근성 → 엣지케이스 → 성능 → 워크플로우 → 기능
  - 카테고리별 결과 요약 출력
  - --filter와 조합 가능

- 내부 리팩터링: runScenarioList() 헬퍼 함수 분리

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
김보곤
2026-02-26 13:14:07 +09:00
parent 851ed29c75
commit 969b119f99
1 changed files with 248 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
e2e/runner/run-all.js
View File

@@ -13,6 +13,9 @@
* node e2e/runner/run-all.js --headless # headless
* node e2e/runner/run-all.js --exclude sales # 파일명에 "sales" 포함된 것 제외
* node e2e/runner/run-all.js --skip-passed # 이미 성공한 시나리오 건너뛰기
* node e2e/runner/run-all.js --iterate # 실패 시나리오 자동 재실행 (최대 3회)
* node e2e/runner/run-all.js --iterate 5 # 실패 시나리오 자동 재실행 (최대 5회)
* node e2e/runner/run-all.js --stage # 카테고리별 단계 실행 (accessibility → edge → perf → workflow → functional)
*/
const fs = require('fs');
@@ -46,6 +49,16 @@ const EXCLUDE = (() => {
return idx >= 0 && args[idx + 1] ? args[idx + 1] : null;
})();
const SKIP_PASSED = args.includes('--skip-passed');
const ITERATE = args.includes('--iterate');
const MAX_ITERATIONS = (() => {
const idx = args.indexOf('--iterate');
if (idx >= 0 && args[idx + 1] && !args[idx + 1].startsWith('--')) {
const n = parseInt(args[idx + 1], 10);
return isNaN(n) ? 3 : Math.min(Math.max(n, 1), 10);
}
return 3;
})();
const STAGE_MODE = args.includes('--stage');
// ─── Helpers ────────────────────────────────────────────────
@@ -1270,6 +1283,8 @@ async function main() {
if (FILTER) console.log(`필터: ${FILTER}`);
if (EXCLUDE) console.log(`제외: ${EXCLUDE}`);
if (SKIP_PASSED) console.log(`스킵: 이미 성공한 시나리오 건너뛰기 (--skip-passed)`);
if (ITERATE) console.log(`반복 모드: 실패 시나리오 자동 재실행 (최대 ${MAX_ITERATIONS}회)`);
if (STAGE_MODE) console.log(`단계 모드: 카테고리별 순차 실행`);
console.log('');
// Ensure directories
@@ -1371,24 +1386,40 @@ async function main() {
process.exit(1);
}
// Run scenarios
const allResults = [];
const startTime = Date.now();
// ─── Helper: Run a list of scenario files and return results ────
async function runScenarioList(files, label, counter) {
const results = [];
for (let i = 0; i < files.length; i++) {
const file = files[i];
const scenarioPath = path.join(SCENARIOS_DIR, file);
counter.current++;
const num = `(${counter.current}/${counter.total})`;
for (let i = 0; i < scenarioFiles.length; i++) {
const file = scenarioFiles[i];
const scenarioPath = path.join(SCENARIOS_DIR, file);
const num = `(${i + 1}/${totalScenarios})`;
process.stdout.write(`${C.dim(num)} ${label ? `[${label}] ` : ''}${file.replace('.json', '')} ... `);
process.stdout.write(`${C.dim(num)} ${file.replace('.json', '')} ... `);
let result;
const timeout = getScenarioTimeout(file);
try {
// Wrap with timeout
result = await Promise.race([
runScenario(page, scenarioPath),
sleep(timeout).then(() => ({
let result;
const timeout = getScenarioTimeout(file);
try {
result = await Promise.race([
runScenario(page, scenarioPath),
sleep(timeout).then(() => ({
id: file.replace('.json', ''),
name: file.replace('.json', ''),
steps: [],
passed: 0,
failed: 0,
warned: 0,
totalSteps: 0,
apiSummary: null,
error: `Timeout (>${timeout / 1000}s)`,
stoppedReason: 'timeout',
currentUrl: '',
startTime: Date.now() - timeout,
endTime: Date.now(),
})),
]);
} catch (scenarioErr) {
result = {
id: file.replace('.json', ''),
name: file.replace('.json', ''),
steps: [],
@@ -1397,66 +1428,213 @@ async function main() {
warned: 0,
totalSteps: 0,
apiSummary: null,
error: `Timeout (>${timeout / 1000}s)`,
stoppedReason: 'timeout',
error: scenarioErr.message,
stoppedReason: 'exception',
currentUrl: '',
startTime: Date.now() - timeout,
startTime: Date.now(),
endTime: Date.now(),
})),
]);
} catch (scenarioErr) {
result = {
id: file.replace('.json', ''),
name: file.replace('.json', ''),
steps: [],
passed: 0,
failed: 0,
warned: 0,
totalSteps: 0,
apiSummary: null,
error: scenarioErr.message,
stoppedReason: 'exception',
currentUrl: '',
startTime: Date.now(),
endTime: Date.now(),
};
};
}
results.push(result);
// Save report
const ts = getTimestamp();
saveReport(result, ts);
// Console output
const hasFail = result.failed > 0 || result.error;
const duration = ((result.endTime - result.startTime) / 1000).toFixed(1);
if (hasFail) {
console.log(`${C.red('FAIL')} ${C.dim(`(${result.passed}/${result.totalSteps} passed, ${duration}s)`)}`);
} else {
console.log(`${C.green('PASS')} ${C.dim(`(${result.passed}/${result.totalSteps}, ${duration}s)`)}`);
}
// After login scenario or if logged out, re-login
if (file === 'login.json' || page.url().includes('/login')) {
try {
await page.goto(`${BASE_URL}/ko/login`, { waitUntil: 'domcontentloaded', timeout: 10000 });
await sleep(500);
await page.fill('#userId', AUTH.username);
await page.fill('#password', AUTH.password);
await page.click("button[type='submit']");
await sleep(3000);
} catch (reloginErr) {
// Continue anyway
}
}
}
return results;
}
// ─── Stage Mode: Category-by-category execution ────────────
const STAGE_ORDER = ['accessibility', 'edge-case', 'performance', 'workflow', 'functional'];
const STAGE_NAMES = {
'accessibility': '접근성 검사',
'edge-case': '엣지 케이스',
'performance': '성능 테스트',
'workflow': '비즈니스 워크플로우',
'functional': '기능 테스트',
};
// Run scenarios
const allResults = [];
const iterationHistory = []; // Track iteration progress
const startTime = Date.now();
if (STAGE_MODE) {
// ─── STAGE MODE: Run by category ─────────────────────────
const categorized = {};
for (const file of scenarioFiles) {
const cat = getScenarioCategory(file);
if (!categorized[cat]) categorized[cat] = [];
categorized[cat].push(file);
}
allResults.push(result);
let stageNum = 0;
const counter = { current: 0, total: totalScenarios };
// Save report
const ts = getTimestamp();
saveReport(result, ts);
for (const cat of STAGE_ORDER) {
const files = categorized[cat];
if (!files || files.length === 0) continue;
// Console output
const hasFail = result.failed > 0 || result.error;
const duration = ((result.endTime - result.startTime) / 1000).toFixed(1);
if (hasFail) {
console.log(`${C.red('FAIL')} ${C.dim(`(${result.passed}/${result.totalSteps} passed, ${duration}s)`)}`);
} else {
console.log(`${C.green('PASS')} ${C.dim(`(${result.passed}/${result.totalSteps}, ${duration}s)`)}`);
stageNum++;
const catName = STAGE_NAMES[cat] || cat;
console.log(C.bold(`\n── Stage ${stageNum}: ${catName} (${files.length}개) ──`));
const stageResults = await runScenarioList(files, catName, counter);
allResults.push(...stageResults);
// Stage summary
const stagePassed = stageResults.filter(r => !r.error && r.failed === 0).length;
const stageFailed = stageResults.length - stagePassed;
const stageRate = stageResults.length > 0 ? Math.round((stagePassed / stageResults.length) * 100) : 0;
console.log(C.cyan(`\n ▸ ${catName} 결과: ${stagePassed}/${stageResults.length} PASS (${stageRate}%)`));
if (stageFailed > 0) {
const failNames = stageResults.filter(r => r.failed > 0 || r.error).map(r => r.id);
console.log(C.red(` ▸ 실패: ${failNames.join(', ')}`));
}
}
// After login scenario or if logged out, re-login
if (file === 'login.json' || page.url().includes('/login')) {
try {
await page.goto(`${BASE_URL}/ko/login`, { waitUntil: 'domcontentloaded', timeout: 10000 });
await sleep(500);
await page.fill('#userId', AUTH.username);
await page.fill('#password', AUTH.password);
await page.click("button[type='submit']");
await sleep(3000);
} catch (reloginErr) {
// Continue anyway
} else {
// ─── NORMAL MODE: Run all sequentially ───────────────────
const counter = { current: 0, total: totalScenarios };
const results = await runScenarioList(scenarioFiles, '', counter);
allResults.push(...results);
}
// ─── Iteration 0 (initial run) summary ───────────────────────
const initialPassed = allResults.filter(r => !r.error && r.failed === 0).length;
const initialFailed = allResults.length - initialPassed;
iterationHistory.push({
iteration: 0,
label: '초기 실행',
total: allResults.length,
passed: initialPassed,
failed: initialFailed,
rate: allResults.length > 0 ? Math.round((initialPassed / allResults.length) * 100) : 0,
fixed: [],
stillFailing: allResults.filter(r => r.failed > 0 || r.error).map(r => r.id),
});
// ─── ITERATE MODE: Re-run failed scenarios ─────────────────
if (ITERATE && initialFailed > 0) {
for (let iteration = 1; iteration <= MAX_ITERATIONS; iteration++) {
const failedIds = allResults.filter(r => r.failed > 0 || r.error).map(r => r.id);
if (failedIds.length === 0) {
console.log(C.green(C.bold(`\n🎯 반복 ${iteration} 불필요: 모든 시나리오 PASS!`)));
break;
}
const failedFiles = failedIds.map(id => `${id}.json`).filter(f => fs.existsSync(path.join(SCENARIOS_DIR, f)));
if (failedFiles.length === 0) break;
console.log(C.bold(C.yellow(`\n═══ 반복 ${iteration}/${MAX_ITERATIONS}: ${failedFiles.length}개 실패 시나리오 재실행 ═══`)));
console.log(C.dim(` 대상: ${failedIds.join(', ')}\n`));
const counter = { current: 0, total: failedFiles.length };
const retryResults = await runScenarioList(failedFiles, `재시도 ${iteration}`, counter);
// Update allResults with retry results
const fixedInThisIteration = [];
for (const retryResult of retryResults) {
const idx = allResults.findIndex(r => r.id === retryResult.id);
if (idx >= 0) {
const wasFail = allResults[idx].failed > 0 || allResults[idx].error;
const nowPass = retryResult.failed === 0 && !retryResult.error;
if (wasFail && nowPass) {
fixedInThisIteration.push(retryResult.id);
}
allResults[idx] = retryResult; // Replace with latest result
}
}
// Iteration summary
const iterPassed = allResults.filter(r => !r.error && r.failed === 0).length;
const iterFailed = allResults.length - iterPassed;
const stillFailing = allResults.filter(r => r.failed > 0 || r.error).map(r => r.id);
iterationHistory.push({
iteration,
label: `반복 ${iteration}`,
total: allResults.length,
passed: iterPassed,
failed: iterFailed,
rate: allResults.length > 0 ? Math.round((iterPassed / allResults.length) * 100) : 0,
fixed: fixedInThisIteration,
stillFailing,
});
console.log(C.cyan(`\n ▸ 반복 ${iteration} 결과: ${iterPassed}/${allResults.length} PASS (${Math.round((iterPassed / allResults.length) * 100)}%)`));
if (fixedInThisIteration.length > 0) {
console.log(C.green(` ▸ 이번 반복에서 수정됨: ${fixedInThisIteration.join(', ')}`));
}
if (stillFailing.length > 0) {
console.log(C.red(` ▸ 여전히 실패: ${stillFailing.join(', ')}`));
} else {
console.log(C.green(C.bold(`\n🎯 모든 시나리오 PASS 달성! (반복 ${iteration}회 만에)`)));
break;
}
}
}
const totalTime = Date.now() - startTime;
// Generate summary report
// Generate summary report (with iteration history)
const summaryTs = getTimestamp();
const summaryMd = generateSummaryReport(allResults, totalTime, summaryTs);
let summaryMd = generateSummaryReport(allResults, totalTime, summaryTs);
// Append iteration history to summary if iterate mode was used
if (ITERATE && iterationHistory.length > 1) {
summaryMd += `\n## 반복 실행 이력\n`;
summaryMd += `| 반복 | 전체 | 성공 | 실패 | 성공률 | 수정됨 | 여전히 실패 |\n`;
summaryMd += `|:----:|:----:|:----:|:----:|:------:|--------|------------|\n`;
for (const h of iterationHistory) {
const fixedStr = h.fixed.length > 0 ? h.fixed.join(', ') : '-';
const failStr = h.stillFailing.length > 0 ? h.stillFailing.join(', ') : '-';
summaryMd += `| ${h.label} | ${h.total} | ${h.passed} | ${h.failed} | ${h.rate}% | ${fixedStr} | ${failStr} |\n`;
}
// Progress bar
summaryMd += `\n### 반복별 추이\n\`\`\`\n`;
for (const h of iterationHistory) {
const barLen = 40;
const filled = Math.round((h.rate / 100) * barLen);
const bar = '█'.repeat(filled) + '░'.repeat(barLen - filled);
summaryMd += `${h.label.padEnd(12)} ${bar} ${h.rate}% (${h.passed}/${h.total})\n`;
}
summaryMd += `\`\`\`\n`;
}
// Append stage summary if stage mode was used
if (STAGE_MODE) {
summaryMd += `\n## 단계별 실행 순서\n`;
summaryMd += `실행 순서: ${STAGE_ORDER.filter(c => allResults.some(r => getScenarioCategory(r.id) === c)).map(c => STAGE_NAMES[c] || c).join(' → ')}\n`;
}
const summaryPath = path.join(RESULTS_DIR, `E2E_FULL_TEST_SUMMARY_${summaryTs}.md`);
fs.writeFileSync(summaryPath, summaryMd, 'utf-8');
@@ -1469,6 +1647,15 @@ async function main() {
console.log(C.bold('\n=== 테스트 완료 ==='));
console.log(`전체: ${totalScenarios} | ${C.green(`성공: ${passCount}`)} | ${failCount > 0 ? C.red(`실패: ${failCount}`) : '실패: 0'}`);
if (ITERATE && iterationHistory.length > 1) {
const lastIter = iterationHistory[iterationHistory.length - 1];
const firstIter = iterationHistory[0];
const improvement = lastIter.passed - firstIter.passed;
if (improvement > 0) {
console.log(C.green(`반복 개선: +${improvement}개 PASS (${firstIter.rate}% → ${lastIter.rate}%)`));
}
console.log(`반복 횟수: ${iterationHistory.length - 1}회`);
}
console.log(`소요 시간: ${(totalTime / 1000 / 60).toFixed(1)}분`);
console.log(`요약 리포트: ${summaryPath}`);
console.log('');