test: E2E 184개 시나리오 전체 테스트 결과 (180 PASS / 4 FAIL, 97.8%)
- run-all.js: 184개 시나리오 순차 실행 러너 고도화 - step-executor.js: 액션 핸들러 확장 및 안정성 개선 - 매출관리 4개 시나리오 실패 원인: 페이지네이션(20행 제한) 환경에서 행수 기반 검증 로직의 구조적 한계 (API 전부 성공, CRUD 동작 정상) → 검색/필터 기반 검증으로 시나리오 수정 필요 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
김보곤
File diff suppressed because it is too large
Load Diff
@@ -450,6 +450,24 @@
|
||||
toggle_switch: 'check',
|
||||
capture: 'capture',
|
||||
screenshot: 'capture',
|
||||
// Workflow (Wave 1)
|
||||
save_context: 'workflow_context_save',
|
||||
load_context: 'workflow_context_load',
|
||||
context_save: 'workflow_context_save',
|
||||
context_load: 'workflow_context_load',
|
||||
verify_cross_module: 'cross_module_verify',
|
||||
// Performance (Wave 2)
|
||||
perf_measure: 'measure_performance',
|
||||
perf_api: 'measure_api_performance',
|
||||
perf_assert: 'assert_performance',
|
||||
performance: 'measure_performance',
|
||||
// Edge cases (Wave 3)
|
||||
boundary_fill: 'fill_boundary',
|
||||
rapid: 'rapid_click',
|
||||
a11y_audit: 'accessibility_audit',
|
||||
accessibility: 'accessibility_audit',
|
||||
keyboard_nav: 'keyboard_navigate',
|
||||
tab_navigate: 'keyboard_navigate',
|
||||
drag_start: 'noop',
|
||||
drag_over: 'noop',
|
||||
drag_end: 'noop',
|
||||
@@ -1523,6 +1541,258 @@
|
||||
return pass(`Menu navigation: ${l1} > ${l2}`);
|
||||
},
|
||||
|
||||
// ── Workflow group (Wave 1) ──
|
||||
async workflow_context_save(action, ctx) {
|
||||
const varName = action.variable || action.key;
|
||||
if (!varName) return fail('workflow_context_save: variable/key required');
|
||||
let value;
|
||||
if (action.selector) {
|
||||
const el = findEl(action.selector, { selectors: ctx.selectors });
|
||||
value = el ? (el.value || el.innerText?.trim() || '') : null;
|
||||
if (!value) return warn(`workflow_context_save: element empty or not found: ${action.selector}`);
|
||||
} else if (action.value) {
|
||||
value = replaceVars(action.value, ctx.variables);
|
||||
} else if (action.extract === 'url_id') {
|
||||
const m = window.location.href.match(/\/(\d+)(?:\?|$)/);
|
||||
value = m ? m[1] : null;
|
||||
if (!value) return warn('workflow_context_save: no ID found in URL');
|
||||
} else if (action.extract === 'first_row_cell') {
|
||||
const rows = document.querySelectorAll('table tbody tr');
|
||||
if (rows.length > 0) {
|
||||
const cellIdx = action.cellIndex ?? 1;
|
||||
const cell = rows[0].querySelectorAll('td')[cellIdx];
|
||||
value = cell?.innerText?.trim() || null;
|
||||
}
|
||||
if (!value) return warn('workflow_context_save: no table data');
|
||||
} else {
|
||||
return fail('workflow_context_save: need selector, value, or extract');
|
||||
}
|
||||
if (!window.__WORKFLOW_CTX__) window.__WORKFLOW_CTX__ = {};
|
||||
window.__WORKFLOW_CTX__[varName] = value;
|
||||
ctx.variables[varName] = value;
|
||||
return pass(`Saved workflow ctx: ${varName}=${String(value).substring(0, 40)}`);
|
||||
},
|
||||
|
||||
async workflow_context_load(action, ctx) {
|
||||
const varName = action.variable || action.key;
|
||||
if (!varName) return fail('workflow_context_load: variable/key required');
|
||||
const value = (window.__WORKFLOW_CTX__ && window.__WORKFLOW_CTX__[varName]) || ctx.variables[varName];
|
||||
if (value === undefined || value === null) return warn(`workflow_context_load: "${varName}" not found in context`);
|
||||
ctx.variables[varName] = value;
|
||||
return pass(`Loaded workflow ctx: ${varName}=${String(value).substring(0, 40)}`);
|
||||
},
|
||||
|
||||
async cross_module_verify(action, ctx) {
|
||||
const searchText = action.search || action.value || ctx.variables[action.variable];
|
||||
if (!searchText) return warn('cross_module_verify: no search text');
|
||||
await sleep(1500);
|
||||
const pageText = document.body.innerText;
|
||||
const found = pageText.includes(searchText);
|
||||
if (action.expect === false) {
|
||||
return found
|
||||
? fail(`cross_module_verify: "${searchText}" should NOT exist`)
|
||||
: pass(`cross_module_verify: correctly absent "${searchText}"`);
|
||||
}
|
||||
return found
|
||||
? pass(`cross_module_verify: "${searchText}" found in target module`)
|
||||
: warn(`cross_module_verify: "${searchText}" NOT found - possible data inconsistency`);
|
||||
},
|
||||
|
||||
// ── Performance group (Wave 2) ──
|
||||
async measure_performance(action, ctx) {
|
||||
const metrics = {};
|
||||
try {
|
||||
const nav = performance.getEntriesByType('navigation')[0];
|
||||
if (nav) {
|
||||
metrics.domContentLoaded = Math.round(nav.domContentLoadedEventEnd - nav.startTime);
|
||||
metrics.load = Math.round(nav.loadEventEnd - nav.startTime);
|
||||
metrics.ttfb = Math.round(nav.responseStart - nav.requestStart);
|
||||
metrics.domInteractive = Math.round(nav.domInteractive - nav.startTime);
|
||||
}
|
||||
const resources = performance.getEntriesByType('resource');
|
||||
metrics.resourceCount = resources.length;
|
||||
metrics.totalTransferKB = Math.round(resources.reduce((s, r) => s + (r.transferSize || 0), 0) / 1024);
|
||||
metrics.domNodes = document.getElementsByTagName('*').length;
|
||||
if (performance.memory) {
|
||||
metrics.jsHeapMB = Math.round(performance.memory.usedJSHeapSize / 1024 / 1024);
|
||||
metrics.heapPct = Math.round((performance.memory.usedJSHeapSize / performance.memory.jsHeapSizeLimit) * 100);
|
||||
}
|
||||
} catch (e) { metrics.error = e.message; }
|
||||
const varName = action.variable || 'perf_metrics';
|
||||
ctx.variables[varName] = JSON.stringify(metrics);
|
||||
if (!window.__PERF_DATA__) window.__PERF_DATA__ = {};
|
||||
window.__PERF_DATA__[window.location.pathname] = metrics;
|
||||
const loadTime = metrics.load || metrics.domContentLoaded || 0;
|
||||
const grade = loadTime < 1000 ? 'A' : loadTime < 2000 ? 'B' : loadTime < 3000 ? 'C' : 'F';
|
||||
return pass(`Perf: load=${loadTime}ms grade=${grade} dom=${metrics.domNodes} res=${metrics.resourceCount}`);
|
||||
},
|
||||
|
||||
async measure_api_performance(action, ctx) {
|
||||
const logs = ApiMonitor._logs.slice();
|
||||
const apiMetrics = {
|
||||
totalCalls: logs.length,
|
||||
avgResponseTime: logs.length > 0 ? Math.round(logs.reduce((s, l) => s + (l.duration || 0), 0) / logs.length) : 0,
|
||||
maxResponseTime: logs.length > 0 ? Math.max(...logs.map(l => l.duration || 0)) : 0,
|
||||
slowCalls: logs.filter(l => l.duration > 2000).length,
|
||||
failedCalls: logs.filter(l => !l.ok).length,
|
||||
calls: logs.slice(-10).map(l => ({ url: l.url.split('?')[0].split('/').slice(-2).join('/'), ms: l.duration, status: l.status })),
|
||||
};
|
||||
const varName = action.variable || 'api_perf';
|
||||
ctx.variables[varName] = JSON.stringify(apiMetrics);
|
||||
return pass(`API: ${apiMetrics.totalCalls} calls, avg=${apiMetrics.avgResponseTime}ms, slow=${apiMetrics.slowCalls}`);
|
||||
},
|
||||
|
||||
async assert_performance(action, ctx) {
|
||||
const thresholds = action.thresholds || {};
|
||||
const maxPageLoad = thresholds.pageLoad || 3000;
|
||||
const maxApiAvg = thresholds.apiAvg || 2000;
|
||||
const maxDomNodes = thresholds.domNodes || 5000;
|
||||
const issues = [];
|
||||
const perfStr = ctx.variables['perf_metrics'];
|
||||
if (perfStr) {
|
||||
try {
|
||||
const m = JSON.parse(perfStr);
|
||||
const loadTime = m.load || m.domContentLoaded || 0;
|
||||
if (loadTime > maxPageLoad) issues.push(`page load ${loadTime}ms > ${maxPageLoad}ms`);
|
||||
if (m.domNodes > maxDomNodes) issues.push(`DOM nodes ${m.domNodes} > ${maxDomNodes}`);
|
||||
} catch (e) {}
|
||||
}
|
||||
const apiStr = ctx.variables['api_perf'];
|
||||
if (apiStr) {
|
||||
try {
|
||||
const a = JSON.parse(apiStr);
|
||||
if (a.avgResponseTime > maxApiAvg) issues.push(`API avg ${a.avgResponseTime}ms > ${maxApiAvg}ms`);
|
||||
} catch (e) {}
|
||||
}
|
||||
if (issues.length > 0) return warn(`Performance: ${issues.join('; ')}`);
|
||||
return pass('Performance within thresholds');
|
||||
},
|
||||
|
||||
// ── Edge case group (Wave 3) ──
|
||||
async fill_boundary(action, ctx) {
|
||||
const el = findEl(action.target, { selectors: ctx.selectors });
|
||||
if (!el) return fail(`Input not found: ${action.target}`);
|
||||
scrollIntoView(el);
|
||||
el.focus();
|
||||
const boundaryType = action.boundaryType || action.boundary || 'empty';
|
||||
let value = '';
|
||||
switch (boundaryType) {
|
||||
case 'empty': value = ''; break;
|
||||
case 'whitespace': value = ' '; break;
|
||||
case 'max_length': value = 'A'.repeat(action.maxLength || 255); break;
|
||||
case 'overflow': value = 'X'.repeat((action.maxLength || 255) + 50); break;
|
||||
case 'special_chars': value = "<script>alert('xss')</script>"; break;
|
||||
case 'sql_injection': value = "'; DROP TABLE users; --"; break;
|
||||
case 'unicode': value = '한글テスト中文🎉'; break;
|
||||
case 'numeric_min': value = String(action.min ?? -999999); break;
|
||||
case 'numeric_max': value = String(action.max ?? 999999999); break;
|
||||
case 'negative': value = '-1'; break;
|
||||
case 'zero': value = '0'; break;
|
||||
case 'decimal': value = '0.123456789'; break;
|
||||
default: value = action.value || '';
|
||||
}
|
||||
clearInput(el);
|
||||
setInputValue(el, value);
|
||||
await sleep(300);
|
||||
return pass(`Boundary fill [${boundaryType}]: "${value.substring(0, 30)}${value.length > 30 ? '...' : ''}"`);
|
||||
},
|
||||
|
||||
async rapid_click(action, ctx) {
|
||||
const el = findEl(action.target, { selectors: ctx.selectors });
|
||||
if (!el) return fail(`Element not found: ${action.target}`);
|
||||
scrollIntoView(el);
|
||||
const clicks = action.count || 5;
|
||||
const delay = action.delay || 50;
|
||||
for (let i = 0; i < clicks; i++) {
|
||||
el.click();
|
||||
if (delay > 0) await sleep(delay);
|
||||
}
|
||||
await sleep(500);
|
||||
return pass(`Rapid clicked ${clicks}x: ${action.target}`);
|
||||
},
|
||||
|
||||
async accessibility_audit(action, ctx) {
|
||||
const issues = { critical: [], serious: [], moderate: [], minor: [] };
|
||||
// Image alt text
|
||||
document.querySelectorAll('img').forEach(img => {
|
||||
if (!img.alt && !img.getAttribute('aria-label') && !img.getAttribute('aria-hidden') && img.offsetParent !== null) {
|
||||
issues.critical.push({ rule: 'image-alt', wcag: '1.1.1', el: img.src?.split('/').pop()?.substring(0, 30) || 'img' });
|
||||
}
|
||||
});
|
||||
// Form labels
|
||||
document.querySelectorAll('input:not([type="hidden"]), select, textarea').forEach(el => {
|
||||
if (el.offsetParent === null) return;
|
||||
const hasLabel = el.id && document.querySelector(`label[for="${el.id}"]`);
|
||||
const hasAria = el.getAttribute('aria-label') || el.getAttribute('aria-labelledby');
|
||||
const hasPlaceholder = el.placeholder;
|
||||
const hasTitle = el.title;
|
||||
if (!hasLabel && !hasAria && !hasPlaceholder && !hasTitle) {
|
||||
issues.critical.push({ rule: 'form-label', wcag: '1.3.1', el: `${el.tagName}[${el.type || 'text'}]` });
|
||||
}
|
||||
});
|
||||
// Button names
|
||||
document.querySelectorAll('button, [role="button"]').forEach(btn => {
|
||||
if (btn.offsetParent === null) return;
|
||||
if (!btn.innerText?.trim() && !btn.getAttribute('aria-label') && !btn.title) {
|
||||
const hasSvg = btn.querySelector('svg');
|
||||
if (!hasSvg || !btn.getAttribute('aria-label')) {
|
||||
issues.serious.push({ rule: 'button-name', wcag: '4.1.2' });
|
||||
}
|
||||
}
|
||||
});
|
||||
// Heading order
|
||||
const headings = Array.from(document.querySelectorAll('h1,h2,h3,h4,h5,h6')).filter(h => h.offsetParent !== null);
|
||||
let prevLevel = 0;
|
||||
headings.forEach(h => {
|
||||
const level = parseInt(h.tagName[1]);
|
||||
if (prevLevel > 0 && level > prevLevel + 1) {
|
||||
issues.moderate.push({ rule: 'heading-order', wcag: '1.3.1', detail: `h${prevLevel}→h${level}` });
|
||||
}
|
||||
prevLevel = level;
|
||||
});
|
||||
// Page language
|
||||
if (!document.documentElement.lang) {
|
||||
issues.moderate.push({ rule: 'html-lang', wcag: '3.1.1' });
|
||||
}
|
||||
// Score
|
||||
const score = Math.max(0, 100 - (issues.critical.length * 10) - (issues.serious.length * 5) - (issues.moderate.length * 2) - (issues.minor.length * 1));
|
||||
const grade = score >= 70 ? 'PASS' : 'FAIL';
|
||||
const result = { score, grade, issues, url: window.location.href };
|
||||
ctx.variables['a11y_result'] = JSON.stringify(result);
|
||||
if (!window.__A11Y_DATA__) window.__A11Y_DATA__ = {};
|
||||
window.__A11Y_DATA__[window.location.pathname] = result;
|
||||
const summary = `A11y: score=${score} ${grade} (C:${issues.critical.length} S:${issues.serious.length} M:${issues.moderate.length})`;
|
||||
return score >= 70 ? pass(summary) : warn(summary);
|
||||
},
|
||||
|
||||
async keyboard_navigate(action, ctx) {
|
||||
const maxTabs = action.count || 20;
|
||||
const focusable = [];
|
||||
let prevActive = document.activeElement;
|
||||
document.body.focus();
|
||||
await sleep(100);
|
||||
for (let i = 0; i < maxTabs; i++) {
|
||||
document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Tab', keyCode: 9, bubbles: true }));
|
||||
document.activeElement?.dispatchEvent?.(new KeyboardEvent('keydown', { key: 'Tab', keyCode: 9, bubbles: true }));
|
||||
await sleep(100);
|
||||
const current = document.activeElement;
|
||||
if (current && current !== document.body && current !== prevActive) {
|
||||
const hasOutline = window.getComputedStyle(current).outlineStyle !== 'none' || window.getComputedStyle(current).boxShadow !== 'none';
|
||||
focusable.push({
|
||||
tag: current.tagName,
|
||||
text: (current.innerText || current.value || '').substring(0, 20),
|
||||
visible: current.offsetParent !== null,
|
||||
focusIndicator: hasOutline,
|
||||
});
|
||||
prevActive = current;
|
||||
}
|
||||
}
|
||||
const withIndicator = focusable.filter(f => f.focusIndicator).length;
|
||||
const allVisible = focusable.every(f => f.visible);
|
||||
ctx.variables['keyboard_nav'] = JSON.stringify({ elements: focusable.length, withIndicator, allVisible });
|
||||
return pass(`Keyboard: ${focusable.length} focusable, ${withIndicator} with indicator, allVisible=${allVisible}`);
|
||||
},
|
||||
|
||||
// ── Noop ──
|
||||
async noop(action, ctx) {
|
||||
return pass('No action');
|
||||
@@ -1581,6 +1851,20 @@
|
||||
return lastResult;
|
||||
}
|
||||
|
||||
// ─── Step Timeout ───────────────────────────────────────
|
||||
|
||||
const STEP_TIMEOUT_MS = 60000; // 60초: 개별 스텝 최대 대기 시간
|
||||
|
||||
/** Wrap a promise with a timeout - 1분 초과 시 즉시 오류 처리 */
|
||||
function withStepTimeout(promise, timeoutMs, stepName) {
|
||||
return Promise.race([
|
||||
promise,
|
||||
new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error(`Step timeout (>${timeoutMs / 1000}s): ${stepName}`)), timeoutMs)
|
||||
),
|
||||
]);
|
||||
}
|
||||
|
||||
// ─── Batch Runner ───────────────────────────────────────
|
||||
|
||||
/**
|
||||
@@ -1609,64 +1893,78 @@
|
||||
let stepError = null;
|
||||
let subResults = [];
|
||||
|
||||
for (let j = 0; j < normalized.subActions.length; j++) {
|
||||
const action = normalized.subActions[j];
|
||||
const actionType = action.type;
|
||||
// Per-step timeout: use step-specific timeout or global 60s
|
||||
const stepTimeoutMs = step.timeout || STEP_TIMEOUT_MS;
|
||||
|
||||
// Check if action requires native (screenshot without selector)
|
||||
if ((actionType === 'capture' && !action.selector) || actionType === 'screenshot') {
|
||||
stoppedReason = 'native_required';
|
||||
stoppedAtIndex = i;
|
||||
results.push({
|
||||
stepId: normalized.stepId,
|
||||
name: normalized.name,
|
||||
status: 'skip',
|
||||
duration: now() - stepStart,
|
||||
details: 'Requires native screenshot',
|
||||
error: null,
|
||||
phase: normalized.phase,
|
||||
});
|
||||
// Return with position info
|
||||
try {
|
||||
await withStepTimeout((async () => {
|
||||
for (let j = 0; j < normalized.subActions.length; j++) {
|
||||
const action = normalized.subActions[j];
|
||||
const actionType = action.type;
|
||||
|
||||
// Check if action requires native (screenshot without selector)
|
||||
if ((actionType === 'capture' && !action.selector) || actionType === 'screenshot') {
|
||||
stoppedReason = 'native_required';
|
||||
stoppedAtIndex = i;
|
||||
results.push({
|
||||
stepId: normalized.stepId,
|
||||
name: normalized.name,
|
||||
status: 'skip',
|
||||
duration: now() - stepStart,
|
||||
details: 'Requires native screenshot',
|
||||
error: null,
|
||||
phase: normalized.phase,
|
||||
});
|
||||
return '__EARLY_RETURN__';
|
||||
}
|
||||
|
||||
// Get handler
|
||||
const handler = ActionHandlers[actionType];
|
||||
if (!handler) {
|
||||
subResults.push(warn(`Unknown action type: ${actionType}`));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Execute with retry
|
||||
const result = await retryAction(handler, action, ctx);
|
||||
|
||||
// Handle navigation signal
|
||||
if (result.status === 'navigation') {
|
||||
subResults.push(result);
|
||||
stoppedReason = 'navigation';
|
||||
stoppedAtIndex = i + 1; // continue from next step
|
||||
const sr = buildStepResult(normalized, subResults, stepStart);
|
||||
results.push(sr);
|
||||
return '__EARLY_RETURN__';
|
||||
}
|
||||
|
||||
// Handle native_required signal
|
||||
if (result.status === 'native_required') {
|
||||
stoppedReason = 'native_required';
|
||||
stoppedAtIndex = i;
|
||||
const sr = buildStepResult(normalized, subResults, stepStart);
|
||||
results.push(sr);
|
||||
return '__EARLY_RETURN__';
|
||||
}
|
||||
|
||||
subResults.push(result);
|
||||
|
||||
// Check URL change (indicates page navigation)
|
||||
const currentUrl = window.location.href;
|
||||
if (currentUrl !== startUrl && j < normalized.subActions.length - 1) {
|
||||
// URL changed mid-step, might need re-injection
|
||||
// Continue for now, check at step boundary
|
||||
}
|
||||
}
|
||||
})(), stepTimeoutMs, normalized.name);
|
||||
|
||||
// Check if early return was signaled
|
||||
if (stoppedReason !== 'complete') {
|
||||
return buildBatchResult(results, ctx, stoppedReason, stoppedAtIndex);
|
||||
}
|
||||
|
||||
// Get handler
|
||||
const handler = ActionHandlers[actionType];
|
||||
if (!handler) {
|
||||
subResults.push(warn(`Unknown action type: ${actionType}`));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Execute with retry
|
||||
const result = await retryAction(handler, action, ctx);
|
||||
|
||||
// Handle navigation signal
|
||||
if (result.status === 'navigation') {
|
||||
subResults.push(result);
|
||||
stoppedReason = 'navigation';
|
||||
stoppedAtIndex = i + 1; // continue from next step
|
||||
const sr = buildStepResult(normalized, subResults, stepStart);
|
||||
results.push(sr);
|
||||
return buildBatchResult(results, ctx, stoppedReason, stoppedAtIndex);
|
||||
}
|
||||
|
||||
// Handle native_required signal
|
||||
if (result.status === 'native_required') {
|
||||
stoppedReason = 'native_required';
|
||||
stoppedAtIndex = i;
|
||||
const sr = buildStepResult(normalized, subResults, stepStart);
|
||||
results.push(sr);
|
||||
return buildBatchResult(results, ctx, stoppedReason, stoppedAtIndex);
|
||||
}
|
||||
|
||||
subResults.push(result);
|
||||
|
||||
// Check URL change (indicates page navigation)
|
||||
const currentUrl = window.location.href;
|
||||
if (currentUrl !== startUrl && j < normalized.subActions.length - 1) {
|
||||
// URL changed mid-step, might need re-injection
|
||||
// Continue for now, check at step boundary
|
||||
}
|
||||
} catch (timeoutErr) {
|
||||
// Step exceeded timeout - record as fail and continue to next step
|
||||
subResults.push(fail(timeoutErr.message));
|
||||
}
|
||||
|
||||
// Build step result
|
||||
|
||||
Reference in New Issue
Block a user