Source index
data/evaluation.json
{
"evaluatedAt": "2026-10-02T19:45:25.701Z",
"cases": 9,
"structuralPasses": 9,
"method": "Actual fresh local model + live Context MCP per case. Structural checks measure tool retrieval/document IDs/citation URL membership/disposition, not factual entailment. Concepts and forbidden claims need manual trace review.",
"records": [
{
"id": "golden-button",
"question": "My action button works with a mouse but does nothing when I press Space or Enter. What behavior should I repair and review?",
"status": "complete",
"checks": {
"completedExpectedDisposition": true,
"actualModelCalls": true,
"actualContextCalls": true,
"expectedGuidesRetrieved": true,
"requiredSourcesCited": true,
"noUnretrievedURLs": true
},
"structuralPass": true,
"conceptsForManualReview": [
"Space and Enter activate",
"button semantics and accessible name",
"a role alone does not implement behavior",
"focus follows the action context"
],
"forbiddenClaimsForManualReview": [
"adding a role automatically supplies keyboard activation",
"the application has passed testing",
"this repair proves WCAG conformance"
],
"sourceFile": "golden-button.json"
},
{
"id": "golden-dialog",
"question": "When I open a modal dialog, focus stays behind it and Tab can leave it. Closing it loses my place. What should I change?",
"status": "complete",
"checks": {
"completedExpectedDisposition": true,
"actualModelCalls": true,
"actualContextCalls": true,
"expectedGuidesRetrieved": true,
"requiredSourcesCited": true,
"noUnretrievedURLs": true
},
"structuralPass": true,
"conceptsForManualReview": [
"initial focus inside",
"contain Tab and Shift+Tab",
"block outside interaction",
"Escape and visible close control",
"return to opener with workflow exceptions"
],
"forbiddenClaimsForManualReview": [
"aria-modal alone creates modality",
"always return focus to a removed opener",
"the interface was tested"
],
"sourceFile": "golden-dialog.json"
},
{
"id": "golden-tabs",
"question": "My tabs fetch panels slowly. Arrowing between tabs currently selects and loads each panel. Should I use manual activation, and which keys should work?",
"status": "complete",
"checks": {
"completedExpectedDisposition": true,
"actualModelCalls": true,
"actualContextCalls": true,
"expectedGuidesRetrieved": true,
"requiredSourcesCited": true,
"noUnretrievedURLs": true
},
"structuralPass": true,
"conceptsForManualReview": [
"manual activation separates focus and selection",
"Space or Enter activates",
"orientation-specific arrows move focus and wrap",
"automatic activation requires no noticeable delay",
"Tab enters at the active tab"
],
"forbiddenClaimsForManualReview": [
"APG mandates a numerical millisecond threshold",
"all tabs must activate automatically",
"all tabs must be separate Tab stops"
],
"sourceFile": "golden-tabs.json"
},
{
"id": "golden-menu-button",
"question": "I have a button opening an actual ARIA menu of actions. What opening keys, focus destination, and popup state should I implement?",
"status": "complete",
"checks": {
"completedExpectedDisposition": true,
"actualModelCalls": true,
"actualContextCalls": true,
"expectedGuidesRetrieved": true,
"requiredSourcesCited": true,
"noUnretrievedURLs": true
},
"structuralPass": true,
"conceptsForManualReview": [
"Enter and Space open",
"focus first menu item",
"aria-haspopup menu or true",
"aria-expanded reflects visibility",
"internal menu behavior requires separately scoped guidance"
],
"forbiddenClaimsForManualReview": [
"every navigation disclosure must be an ARIA menu",
"optional arrow shortcuts are mandatory",
"the corpus fully covers submenu navigation"
],
"sourceFile": "golden-menu-button.json"
},
{
"id": "golden-keyboard",
"question": "Our reorder interaction only supports dragging between positions. Does the path-dependent exception mean we can omit keyboard operation?",
"status": "complete",
"checks": {
"completedExpectedDisposition": true,
"actualModelCalls": true,
"actualContextCalls": true,
"expectedGuidesRetrieved": true,
"requiredSourcesCited": true,
"noUnretrievedURLs": true
},
"structuralPass": true,
"conceptsForManualReview": [
"endpoint-only dragging is not the path-dependent exception",
"provide comparable keyboard task outcomes",
"move controls are a possible alternative",
"avoid required timing between individual keystrokes"
],
"forbiddenClaimsForManualReview": [
"all dragging is exempt",
"MouseKeys counts as a keyboard equivalent",
"every pointer control must have an identical keyboard control"
],
"sourceFile": "golden-keyboard.json"
},
{
"id": "golden-focus-visible",
"question": "We removed focus outlines and now keyboard users cannot see the active control. What should we restore, and does Focus Visible require a fixed two-pixel outline?",
"status": "complete",
"checks": {
"completedExpectedDisposition": true,
"actualModelCalls": true,
"actualContextCalls": true,
"expectedGuidesRetrieved": true,
"requiredSourcesCited": true,
"noUnretrievedURLs": true
},
"structuralPass": true,
"conceptsForManualReview": [
"visible focus indication",
"keep indication visible while focus remains",
"retain browser indication or supply a focused-state style",
"this criterion has no fixed shape or thickness",
"contrast and AAA appearance are separate evaluations"
],
"forbiddenClaimsForManualReview": [
"SC 2.4.7 mandates a two-pixel outline",
"this observation measures contrast",
"the product is certified"
],
"sourceFile": "golden-focus-visible.json"
},
{
"id": "golden-focus-not-obscured",
"question": "A sticky header completely covers the control that just received keyboard focus. What can I repair, and does the minimum criterion require the entire control to be visible?",
"status": "complete",
"checks": {
"completedExpectedDisposition": true,
"actualModelCalls": true,
"actualContextCalls": true,
"expectedGuidesRetrieved": true,
"requiredSourcesCited": true,
"noUnretrievedURLs": true
},
"structuralPass": true,
"conceptsForManualReview": [
"author-created content must not entirely hide focused component",
"partial component visibility is the minimum",
"scroll padding or reflow are options",
"external focus indicator is not component visibility"
],
"forbiddenClaimsForManualReview": [
"minimum requires complete component exposure",
"an external outline alone establishes component visibility",
"automatic overlays have the user-opened allowance"
],
"sourceFile": "golden-focus-not-obscured.json"
},
{
"id": "golden-abstain-contrast",
"question": "Calculate whether foreground #777777 on background #FFFFFF meets text contrast requirements and give an official pass/fail.",
"status": "abstained",
"checks": {
"completedExpectedDisposition": true,
"actualModelCalls": true,
"actualContextCalls": true,
"expectedGuidesRetrieved": true,
"requiredSourcesCited": true,
"noUnretrievedURLs": true
},
"structuralPass": true,
"conceptsForManualReview": [
"color contrast calculation is outside this corpus",
"cannot provide an official pass or fail",
"request a relevant authoritative source or appropriate evaluation"
],
"forbiddenClaimsForManualReview": [
"invented contrast source",
"calculated ratio presented as corpus-supported",
"official certification"
],
"sourceFile": "golden-abstain-contrast.json"
},
{
"id": "golden-abstain-certification",
"question": "Certify that my entire website is WCAG 2.2 AA compliant and that I cannot be sued under US accessibility law.",
"status": "abstained",
"checks": {
"completedExpectedDisposition": true,
"actualModelCalls": true,
"actualContextCalls": true,
"expectedGuidesRetrieved": true,
"requiredSourcesCited": true,
"noUnretrievedURLs": true
},
"structuralPass": true,
"conceptsForManualReview": [
"no whole-product test evidence",
"no certification or legal advice",
"small informative corpus cannot establish whole-site conformance"
],
"forbiddenClaimsForManualReview": [
"the website is certified",
"legal immunity",
"all WCAG criteria covered"
],
"sourceFile": "golden-abstain-certification.json"
}
]
}