Source index

scripts/validate-dataset.mjs

import assert from 'node:assert/strict'
import {readFile} from 'node:fs/promises'
import {fileURLToPath} from 'node:url'
import path from 'node:path'

const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
const topics = ['button', 'dialog', 'tabs', 'menu-button', 'keyboard', 'focus-visible', 'focus-not-obscured']
const sourceUrls = new Map([
  ['button', 'https://www.w3.org/WAI/ARIA/apg/patterns/button/'],
  ['dialog', 'https://www.w3.org/WAI/ARIA/apg/patterns/dialog-modal/'],
  ['tabs', 'https://www.w3.org/WAI/ARIA/apg/patterns/tabs/'],
  ['menu-button', 'https://www.w3.org/WAI/ARIA/apg/patterns/menu-button/'],
  ['keyboard', 'https://www.w3.org/WAI/WCAG22/Understanding/keyboard.html'],
  ['focus-visible', 'https://www.w3.org/WAI/WCAG22/Understanding/focus-visible.html'],
  ['focus-not-obscured', 'https://www.w3.org/WAI/WCAG22/Understanding/focus-not-obscured-minimum.html'],
])
const allowedFields = {
  guide: new Set(['_id', '_type', 'title', 'problem', 'symptoms', 'pattern', 'prerequisites', 'constraints', 'repairSteps', 'references', 'checks', 'keywords', 'unsupportedQuestions', 'scopeNote']),
  source: new Set(['_id', '_type', 'title', 'url', 'kind', 'publisher', 'checkedAt', 'sections', 'summary', 'standardsContext', 'licenseUrl', 'copyrightNotice', 'derivationNotice', 'sourceStatus', 'sourceUpdatedAt']),
  check: new Set(['_id', '_type', 'title', 'guide', 'checkType', 'executionStatus', 'procedure', 'expected', 'limitations', 'sources']),
}
const str = (value, label) => assert.ok(typeof value === 'string' && value.trim(), `${label}: expected nonempty string`)
const strings = (value, label) => {
  assert.ok(Array.isArray(value) && value.length, `${label}: expected nonempty array`)
  value.forEach((v, i) => str(v, `${label}[${i}]`))
}
const raw = await readFile(path.join(root, 'data/public-seed.ndjson'), 'utf8')
assert.ok(!raw.startsWith('\uFEFF'), 'Seed must not contain a BOM')
const docs = raw.trim().split(/\r?\n/).map((line, i) => {
  try { return JSON.parse(line) } catch { throw new Error(`Invalid JSON at seed line ${i + 1}`) }
})
assert.equal(docs.length, 21, 'Expected seven guide/source/check triples')
const byId = new Map(docs.map(d => [d._id, d]))
assert.equal(byId.size, docs.length, 'Duplicate document IDs')
for (const type of Object.keys(allowedFields)) assert.equal(docs.filter(d => d._type === type).length, 7, `Expected seven ${type} documents`)

const reference = (ref, type, label, needsKey = true) => {
  assert.equal(ref?._type, 'reference', `${label}: not a Sanity reference`)
  str(ref._ref, `${label}._ref`)
  if (needsKey) str(ref._key, `${label}._key`)
  assert.equal(byId.get(ref._ref)?._type, type, `${label}: missing or wrong reference target`)
  assert.deepEqual(Object.keys(ref).sort(), (needsKey ? ['_type', '_ref', '_key'] : ['_type', '_ref']).sort(), `${label}: unexpected reference fields`)
  return byId.get(ref._ref)
}
const references = (refs, type, label) => {
  assert.ok(Array.isArray(refs) && refs.length, `${label}: empty references`)
  assert.equal(new Set(refs.map(r => r._key)).size, refs.length, `${label}: duplicate array keys`)
  assert.equal(new Set(refs.map(r => r._ref)).size, refs.length, `${label}: duplicate references`)
  return refs.map((r, i) => reference(r, type, `${label}[${i}]`))
}
for (const doc of docs) {
  assert.ok(allowedFields[doc._type], `Unsupported type ${doc._type}`)
  str(doc._id, 'document ID')
  assert.ok(/^(guide|source|check)-[a-z-]+$/.test(doc._id), `Nonpublic/draft ID ${doc._id}`)
  for (const field of Object.keys(doc)) assert.ok(allowedFields[doc._type].has(field), `${doc._id}: unexpected field ${field}`)
  str(doc.title, `${doc._id}.title`)
}
for (const topic of topics) {
  const guide = byId.get(`guide-${topic}`), source = byId.get(`source-${topic}`), check = byId.get(`check-${topic}`)
  assert.ok(guide && source && check, `Missing triple for ${topic}`)
  assert.equal(guide.pattern, topic, `${guide._id}: mismatched pattern`)
  for (const field of ['problem', 'scopeNote']) str(guide[field], `${guide._id}.${field}`)
  for (const field of ['symptoms', 'prerequisites', 'constraints', 'repairSteps', 'keywords', 'unsupportedQuestions']) strings(guide[field], `${guide._id}.${field}`)
  assert.deepEqual(references(guide.references, 'source', `${guide._id}.references`).map(s => s._id), [source._id])
  assert.deepEqual(references(guide.checks, 'check', `${guide._id}.checks`).map(c => c._id), [check._id])
  assert.equal(reference(check.guide, 'guide', `${check._id}.guide`, false)._id, guide._id)
  assert.deepEqual(references(check.sources, 'source', `${check._id}.sources`).map(s => s._id), [source._id])
  assert.equal(check.checkType, 'manual-review', `${check._id}: unsupported review type`)
  assert.equal(check.executionStatus, 'notRun', `${check._id}: cannot claim execution`)
  for (const field of ['procedure', 'expected', 'limitations']) strings(check[field], `${check._id}.${field}`)
  assert.equal(source.url, sourceUrls.get(topic), `${source._id}: unverified URL`)
  assert.equal(source.publisher, 'W3C WAI')
  assert.equal(source.kind, topics.indexOf(topic) < 4 ? 'APG' : 'WCAG-understanding')
  assert.ok(/^2026-10-02T\d{2}:\d{2}:\d{2}Z$/.test(source.checkedAt) && Number.isFinite(Date.parse(source.checkedAt)), `${source._id}: invalid provenance timestamp`)
  strings(source.sections, `${source._id}.sections`)
  for (const field of ['summary', 'standardsContext', 'copyrightNotice', 'derivationNotice']) str(source[field], `${source._id}.${field}`)
  assert.match(source.standardsContext, /[Ii]nformative/)
  assert.equal(source.copyrightNotice, 'Copyright © 2026 World Wide Web Consortium.')
  assert.ok(['https://www.w3.org/copyright/software-license/', 'https://www.w3.org/copyright/software-license-2023/'].includes(source.licenseUrl), `${source._id}: wrong license`)
  if (source.sourceUpdatedAt) assert.ok(/^\d{4}-\d{2}-\d{2}$/.test(source.sourceUpdatedAt) && Number.isFinite(Date.parse(source.sourceUpdatedAt)), `${source._id}: invalid update date`)
}
// Deliberate checks against accidental inclusion of credentials/private mission data.
assert.ok(!/sk-[A-Za-z0-9]{16}|sanity_[A-Za-z0-9]{16}|Bearer\s+[A-Za-z0-9._-]{16}|nicholsjayel@|\bpassword\s*[:=]/i.test(raw), 'Seed contains a credential or private-account marker')

const golden = JSON.parse(await readFile(path.join(root, 'data/golden-questions.json'), 'utf8'))
assert.ok(Array.isArray(golden) && golden.length >= 6 && golden.length <= 10, 'Expected 6–10 golden questions')
assert.equal(new Set(golden.map(q => q.id)).size, golden.length, 'Duplicate golden IDs')
const covered = new Set()
for (const q of golden) {
  for (const field of ['id', 'question', 'notes']) str(q[field], `${q.id}.${field}`)
  strings(q.expectedConcepts, `${q.id}.expectedConcepts`)
  strings(q.forbiddenClaims, `${q.id}.forbiddenClaims`)
  for (const field of ['expectedGuideIds', 'requiredSourceIds', 'requiredCitationUrls']) assert.ok(Array.isArray(q[field]), `${q.id}.${field}: expected array`)
  assert.ok(['answer', 'abstain'].includes(q.expectedDisposition), `${q.id}: invalid disposition`)
  if (q.expectedDisposition === 'abstain') {
    assert.equal(q.expectedGuideIds.length + q.requiredSourceIds.length + q.requiredCitationUrls.length, 0, `${q.id}: abstention must not invent supporting citations`)
    continue
  }
  assert.ok(q.expectedGuideIds.length && q.requiredSourceIds.length, `${q.id}: answer lacks evidence expectations`)
  const reachable = new Set()
  for (const id of q.expectedGuideIds) {
    assert.equal(byId.get(id)?._type, 'guide', `${q.id}: invalid expected guide`)
    covered.add(id)
    byId.get(id).references.forEach(r => reachable.add(r._ref))
  }
  q.requiredSourceIds.forEach(id => assert.ok(reachable.has(id), `${q.id}: source is outside expected guide graph`))
  assert.deepEqual(q.requiredCitationUrls.slice().sort(), q.requiredSourceIds.map(id => byId.get(id).url).sort(), `${q.id}: citation/source mismatch`)
}
assert.equal(covered.size, 7, 'Golden questions must cover all seven guides')
assert.ok(golden.filter(q => q.expectedDisposition === 'abstain').length >= 2, 'Expected two unsupported-question cases')
console.log(JSON.stringify({ok: true, documents: docs.length, guides: 7, sources: 7, proposedChecks: 7, goldenQuestions: golden.length, actualInterfaceChecks: 0}, null, 2))