GREACT/KIS-TOiR
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(llm-first): context budget, validation, and eval harness, orchestration general-prompt

Browse Source
This commit is contained in:
MaKarin
2026-04-03 14:17:21 +03:00
parent 79c9589658
commit c42a88dff6
189 changed files with 15538 additions and 9109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

301
tools/api-summary-to-openapi.mjs Normal file
View File

@@ -0,0 +1,301 @@
// Deterministic OpenAPI 3.0.3 generator from api-summary.json / toir.api.dsl.
//
// This script is part of the Tier 1 deterministic preprocessing layer.
// It converts the canonical api-summary (produced by tools/api-summary.mjs)
// into a valid OpenAPI 3.0.3 document.
//
// Usage:
// node tools/api-summary-to-openapi.mjs --out openapi.json
// npm run generate:openapi
//
// No LLM involvement. The output is reproducible from DSL + this script alone.
import { writeFileSync } from 'node:fs';
import path from 'node:path';
import { buildApiSummary } from './api-summary.mjs';
const rootDir = process.cwd();
// ---------------------------------------------------------------------------
// DSL scalar → OpenAPI type
// ---------------------------------------------------------------------------
function dslTypeToOpenApi(dslType) {
switch (dslType) {
case 'uuid':
return { type: 'string', format: 'uuid' };
case 'string':
return { type: 'string' };
case 'text':
return { type: 'string' };
case 'integer':
return { type: 'integer', format: 'int32' };
case 'number':
return { type: 'number' };
case 'decimal':
return { type: 'string', format: 'decimal' };
case 'date':
return { type: 'string', format: 'date-time' };
case 'boolean':
return { type: 'boolean' };
default:
// enum names and DTO references handled by caller
return null;
}
}
// ---------------------------------------------------------------------------
// Resolve a DSL field type to an OpenAPI schema reference or inline schema.
// dtoNames — Set of known DTO names in the api summary.
// enumNames — Set of known enum names (derived from type mappings table).
// ---------------------------------------------------------------------------
function resolveFieldType(dslType, dtoNames, enumNames) {
if (!dslType) return { type: 'object' };
// Array type: "DTO.Foo[]"
if (dslType.endsWith('[]')) {
const inner = dslType.slice(0, -2);
return { type: 'array', items: resolveFieldType(inner, dtoNames, enumNames) };
}
// Scalar
const scalar = dslTypeToOpenApi(dslType);
if (scalar) return scalar;
// DTO reference
if (dtoNames.has(dslType)) {
return { $ref: `#/components/schemas/${dslType.replace(/^DTO\./, '')}` };
}
// Enum reference
if (enumNames.has(dslType)) {
return { $ref: `#/components/schemas/${dslType}` };
}
// Unknown — emit as string with x-dsl-type annotation
return { type: 'string', 'x-dsl-type': dslType };
}
// ---------------------------------------------------------------------------
// Build OpenAPI schema object from a DTO definition
// ---------------------------------------------------------------------------
function buildDtoSchema(dto, dtoNames, enumNames) {
const properties = {};
const required = [];
for (const field of dto.fields) {
const schema = resolveFieldType(field.type, dtoNames, enumNames);
if (field.description && schema.$ref) {
// OpenAPI 3.0.3: $ref cannot have sibling keys — wrap with allOf
properties[field.name] = { allOf: [schema], description: field.description };
} else {
if (field.description) schema.description = field.description;
properties[field.name] = schema;
}
if (field.required) required.push(field.name);
}
const schema = {
type: 'object',
properties,
};
if (dto.description) schema.description = dto.description;
if (required.length > 0) schema.required = required;
return schema;
}
// ---------------------------------------------------------------------------
// Convert DSL HTTP method to OpenAPI method key
// ---------------------------------------------------------------------------
function methodKey(method) {
return (method ?? 'get').toLowerCase();
}
// ---------------------------------------------------------------------------
// Build OpenAPI path item for an endpoint
// ---------------------------------------------------------------------------
function buildPathOperation(endpoint, apiDescription, dtoNames, enumNames) {
const operation = {};
if (endpoint.description) operation.summary = endpoint.description;
// Security — all endpoints require bearer auth
operation.security = [{ bearerAuth: [] }];
// Tags — derive from API name or path
const tag = apiDescription ? apiDescription.replace(/^API управления\s+/i, '').replace(/ами$/, '') : undefined;
if (tag) operation.tags = [tag];
// Parameters — detect path params by matching attribute names against {param} in the path
const pathTemplate = endpoint.path ?? '';
const pathParamNames = new Set(
[...pathTemplate.matchAll(/\{(\w+)\}/g)].map((m) => m[1]),
);
const pathParams = endpoint.attributes.filter((a) => pathParamNames.has(a.name));
if (pathParams.length > 0) {
operation.parameters = pathParams.map((p) => ({
name: p.name,
in: 'path',
required: true,
schema: resolveFieldType(p.type, dtoNames, enumNames),
...(p.description ? { description: p.description } : {}),
}));
}
// Request body
const requestAttr = endpoint.attributes.find((a) => a.name === 'request');
if (requestAttr) {
operation.requestBody = {
required: true,
content: {
'application/json': {
schema: resolveFieldType(requestAttr.type, dtoNames, enumNames),
},
},
};
}
// Response
const responseAttr = endpoint.attributes.find((a) => a.name === 'response');
const responseSchema = responseAttr
? resolveFieldType(responseAttr.type, dtoNames, enumNames)
: { type: 'object' };
const successCode = endpoint.method === 'POST' && !endpoint.path?.endsWith('/page') ? '201' : '200';
operation.responses = {
[successCode]: {
description: 'Success',
content: {
'application/json': {
schema: responseSchema,
},
},
},
'401': { description: 'Unauthorized' },
'403': { description: 'Forbidden' },
};
if (endpoint.method === 'DELETE') {
operation.responses = {
'204': { description: 'No content' },
'401': { description: 'Unauthorized' },
'403': { description: 'Forbidden' },
'404': { description: 'Not found' },
};
delete operation.responses['201'];
}
return operation;
}
// ---------------------------------------------------------------------------
// Main builder
// ---------------------------------------------------------------------------
function buildOpenApi(summary) {
const dtoNames = new Set(summary.dtos.map((d) => d.name));
// Build enum map from api-summary enums block (fully declared enums with values)
const enumMap = new Map((summary.enums ?? []).map((e) => [e.name, e]));
// Also collect enum names referenced in DTO fields that are not in the declared enums
// (covers cases where enums are declared in domain.dsl but referenced in api.dsl)
const enumNames = new Set(enumMap.keys());
for (const dto of summary.dtos) {
for (const field of dto.fields) {
const t = field.type?.replace('[]', '');
if (t && !dtoNames.has(t) && !dslTypeToOpenApi(t)) {
enumNames.add(t);
}
}
}
// Schemas — one per DTO
const schemas = {};
for (const dto of summary.dtos) {
const schemaName = dto.name.replace(/^DTO\./, '');
schemas[schemaName] = buildDtoSchema(dto, dtoNames, enumNames);
}
// Enum schemas — use actual values when available, otherwise annotate as opaque string enum
for (const enumName of enumNames) {
const enumDef = enumMap.get(enumName);
if (enumDef && enumDef.values.length > 0) {
schemas[enumName] = {
type: 'string',
enum: enumDef.values.map((v) => v.name),
'x-enum-labels': Object.fromEntries(
enumDef.values.filter((v) => v.label).map((v) => [v.name, v.label]),
),
...(enumDef.description ? { description: enumDef.description } : {}),
};
} else {
schemas[enumName] = {
type: 'string',
'x-dsl-enum': enumName,
description: `Enum: ${enumName} (values defined in domain/*.api.dsl)`,
};
}
}
// Paths
const paths = {};
for (const api of summary.apis) {
for (const endpoint of api.endpoints) {
if (!endpoint.path) continue;
const pathKey = endpoint.path;
if (!paths[pathKey]) paths[pathKey] = {};
const opKey = methodKey(endpoint.method);
paths[pathKey][opKey] = buildPathOperation(endpoint, api.description, dtoNames, enumNames);
}
}
return {
openapi: '3.0.3',
info: {
title: 'KIS-TOiR API',
description:
'Equipment maintenance management system. Generated from domain/toir.api.dsl via tools/api-summary-to-openapi.mjs.',
version: '1.0.0',
},
servers: [
{
url: '/api',
description: 'Default server',
},
],
components: {
securitySchemes: {
bearerAuth: {
type: 'http',
scheme: 'bearer',
bearerFormat: 'JWT',
},
},
schemas,
},
paths,
};
}
// ---------------------------------------------------------------------------
// CLI
// ---------------------------------------------------------------------------
const args = process.argv.slice(2);
const outIndex = args.indexOf('--out');
const outPath = outIndex !== -1 ? args[outIndex + 1] : 'openapi.json';
const summary = buildApiSummary(rootDir);
const openApiDoc = buildOpenApi(summary);
const outputPath = path.resolve(rootDir, outPath);
writeFileSync(outputPath, `${JSON.stringify(openApiDoc, null, 2)}\n`, 'utf8');
console.log(`Generated ${path.relative(rootDir, outputPath)}`);

389
tools/api-summary.mjs Normal file
View File

@@ -0,0 +1,389 @@
import { readdirSync, readFileSync } from 'node:fs';
import path from 'node:path';
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function stripInlineComment(line) {
let inString = false;
let result = '';
for (let index = 0; index < line.length; index += 1) {
const current = line[index];
const next = line[index + 1];
if (current === '"' && line[index - 1] !== '\\') {
inString = !inString;
result += current;
continue;
}
if (!inString && current === '/' && next === '/') {
break;
}
result += current;
}
return result.trim();
}
// ---------------------------------------------------------------------------
// File discovery
// ---------------------------------------------------------------------------
export function getApiDslFiles(rootDir) {
const domainDir = path.join(rootDir, 'domain');
try {
return readdirSync(domainDir, { withFileTypes: true })
.filter((entry) => entry.isFile() && entry.name.toLowerCase().endsWith('.api.dsl'))
.map((entry) => path.join(domainDir, entry.name))
.sort((left, right) => left.localeCompare(right));
} catch {
return [];
}
}
// ---------------------------------------------------------------------------
// Parser
//
// Parses all *.api.dsl files using a stack-based approach.
// This is the single canonical parser for the API DSL.
//
// Supported top-level blocks:
// enum <Name> { description?; value <Name> { label?; } }
// dto DTO.<Name> { description?; attribute <name> { ... } }
// api API.<Name> { description?; endpoint <name> { ... } }
//
// DTO attribute modifiers (any order inside the attribute block):
// type <type>;
// description "...";
// map Entity.field;
// sync Entity.field; (alias for map — used for computed/aggregate fields)
// is required;
// is nullable;
// is unique;
// key primary;
// label "...";
//
// Endpoint modifiers:
// label "METHOD /path";
// description "...";
// attribute <name> { type <type>; description?; }
//
// Returns:
// {
// files: string[],
// enums: EnumBlock[],
// dtos: DtoBlock[],
// apis: ApiBlock[],
// }
//
// EnumBlock = { name, description, values: EnumValue[] }
// EnumValue = { name, label }
// DtoBlock = { name, description, fields: DtoField[] }
// DtoField = { name, type, required, nullable, unique, primary, description, map, label }
// ApiBlock = { name, description, endpoints: Endpoint[] }
// Endpoint = { name, label, method, path, description, attributes: EndpointAttr[] }
// EndpointAttr = { name, type, description }
// ---------------------------------------------------------------------------
export function parseApiDsl(rootDir) {
const files = getApiDslFiles(rootDir);
const enums = [];
const dtos = [];
const apis = [];
const stack = [];
for (const filePath of files) {
const content = readFileSync(filePath, 'utf8');
const lines = content.split(/\r?\n/);
for (const rawLine of lines) {
const line = stripInlineComment(rawLine);
if (!line) continue;
const top = stack.at(-1);
// ── Top-level: enum block ──────────────────────────────────────────
const enumMatch = line.match(/^enum\s+([A-Za-z][A-Za-z0-9_]*)\s*\{$/);
if (!top && enumMatch) {
const enumBlock = { name: enumMatch[1], description: null, values: [] };
enums.push(enumBlock);
stack.push({ type: 'enum', ref: enumBlock });
continue;
}
// ── Top-level: dto block ───────────────────────────────────────────
const dtoMatch = line.match(/^dto\s+(DTO\.\w+)\s*\{$/);
if (!top && dtoMatch) {
const dto = { name: dtoMatch[1], description: null, fields: [] };
dtos.push(dto);
stack.push({ type: 'dto', ref: dto });
continue;
}
// ── Top-level: api block ───────────────────────────────────────────
const apiMatch = line.match(/^api\s+(API\.\w+)\s*\{$/);
if (!top && apiMatch) {
const api = { name: apiMatch[1], description: null, endpoints: [] };
apis.push(api);
stack.push({ type: 'api', ref: api });
continue;
}
// ── Inside enum ───────────────────────────────────────────────────
if (top?.type === 'enum') {
const descMatch = line.match(/^description\s+"(.*)"\s*;$/);
if (descMatch) {
top.ref.description = descMatch[1];
continue;
}
const valueMatch = line.match(/^value\s+([^\s{]+)\s*\{$/);
if (valueMatch) {
const enumValue = { name: valueMatch[1], label: null };
top.ref.values.push(enumValue);
stack.push({ type: 'enumValue', ref: enumValue });
continue;
}
}
// ── Inside enum value ─────────────────────────────────────────────
if (top?.type === 'enumValue') {
const labelMatch = line.match(/^label\s+"(.*)"\s*;$/);
if (labelMatch) {
top.ref.label = labelMatch[1];
continue;
}
}
// ── Inside dto ────────────────────────────────────────────────────
if (top?.type === 'dto') {
const descMatch = line.match(/^description\s+"(.*)"\s*;$/);
if (descMatch) {
top.ref.description = descMatch[1];
continue;
}
const attrMatch = line.match(/^attribute\s+(\w+)\s*\{$/);
if (attrMatch) {
const field = {
name: attrMatch[1],
type: null,
required: false,
nullable: false,
unique: false,
primary: false,
description: null,
map: null,
label: null,
};
top.ref.fields.push(field);
stack.push({ type: 'dtoField', ref: field });
continue;
}
}
// ── Inside dto attribute field ────────────────────────────────────
if (top?.type === 'dtoField') {
const typeMatch = line.match(/^type\s+(.+?)\s*;$/);
if (typeMatch) {
top.ref.type = typeMatch[1];
continue;
}
if (/^is\s+required\s*;$/.test(line)) {
top.ref.required = true;
continue;
}
if (/^is\s+nullable\s*;$/.test(line)) {
top.ref.nullable = true;
continue;
}
if (/^is\s+unique\s*;$/.test(line)) {
top.ref.unique = true;
continue;
}
if (/^key\s+primary\s*;$/.test(line)) {
top.ref.primary = true;
continue;
}
const descMatch = line.match(/^description\s+"(.*)"\s*;$/);
if (descMatch) {
top.ref.description = descMatch[1];
continue;
}
// map Entity.field; — canonical field mapping
const mapMatch = line.match(/^map\s+(\w+)\.(\w+)\s*;$/);
if (mapMatch) {
top.ref.map = `${mapMatch[1]}.${mapMatch[2]}`;
continue;
}
// sync Entity.field; — aggregate / computed field mapping (treated as map)
const syncMatch = line.match(/^sync\s+(\w+)\.(\w+)\s*;$/);
if (syncMatch) {
top.ref.map = `${syncMatch[1]}.${syncMatch[2]}`;
top.ref.sync = true;
continue;
}
const labelMatch = line.match(/^label\s+"(.*)"\s*;$/);
if (labelMatch) {
top.ref.label = labelMatch[1];
continue;
}
}
// ── Inside api ────────────────────────────────────────────────────
if (top?.type === 'api') {
const descMatch = line.match(/^description\s+"(.*)"\s*;$/);
if (descMatch) {
top.ref.description = descMatch[1];
continue;
}
const epMatch = line.match(/^endpoint\s+(\w+)\s*\{$/);
if (epMatch) {
const ep = {
name: epMatch[1],
label: null,
method: null,
path: null,
description: null,
attributes: [],
};
top.ref.endpoints.push(ep);
stack.push({ type: 'endpoint', ref: ep });
continue;
}
}
// ── Inside endpoint ───────────────────────────────────────────────
if (top?.type === 'endpoint') {
const labelMatch = line.match(/^label\s+"([^"]+)"\s*;$/);
if (labelMatch) {
top.ref.label = labelMatch[1];
const parts = labelMatch[1].split(' ', 2);
top.ref.method = parts[0]?.toUpperCase() ?? null;
top.ref.path = parts[1] ?? null;
continue;
}
const descMatch = line.match(/^description\s+"(.*)"\s*;$/);
if (descMatch) {
top.ref.description = descMatch[1];
continue;
}
const attrMatch = line.match(/^attribute\s+(\w+)\s*\{$/);
if (attrMatch) {
const attr = { name: attrMatch[1], type: null, description: null };
top.ref.attributes.push(attr);
stack.push({ type: 'endpointAttr', ref: attr });
continue;
}
}
// ── Inside endpoint attribute ─────────────────────────────────────
if (top?.type === 'endpointAttr') {
const typeMatch = line.match(/^type\s+(.+?)\s*;$/);
if (typeMatch) {
top.ref.type = typeMatch[1];
continue;
}
const descMatch = line.match(/^description\s+"(.*)"\s*;$/);
if (descMatch) {
top.ref.description = descMatch[1];
continue;
}
}
// ── Closing brace — pop the stack ─────────────────────────────────
if (/^}\s*;?$/.test(line)) {
stack.pop();
}
}
}
return { files, enums, dtos, apis };
}
// ---------------------------------------------------------------------------
// Summary builder
//
// Produces the serialisable api-summary.json object.
// ---------------------------------------------------------------------------
export function buildApiSummary(rootDir) {
const { files, enums, dtos, apis } = parseApiDsl(rootDir);
// Detect duplicate DTO names
const dtoNames = new Set();
for (const dto of dtos) {
if (dtoNames.has(dto.name)) {
throw new Error(`Duplicate DTO definition: ${dto.name}`);
}
dtoNames.add(dto.name);
}
// Detect duplicate API names
const apiNames = new Set();
for (const api of apis) {
if (apiNames.has(api.name)) {
throw new Error(`Duplicate API definition: ${api.name}`);
}
apiNames.add(api.name);
}
return {
sourceFiles: files.map((filePath) =>
path.relative(rootDir, filePath).replaceAll('\\', '/'),
),
enums: enums.map((e) => ({
name: e.name,
description: e.description,
values: e.values.map((v) => ({ name: v.name, label: v.label })),
})),
dtos: dtos.map((dto) => ({
name: dto.name,
description: dto.description,
fields: dto.fields.map((field) => ({
name: field.name,
type: field.type,
required: field.required,
nullable: field.nullable,
unique: field.unique,
primary: field.primary,
description: field.description,
map: field.map,
sync: field.sync ?? false,
label: field.label,
})),
})),
apis: apis.map((api) => ({
name: api.name,
description: api.description,
endpoints: api.endpoints.map((ep) => ({
name: ep.name,
label: ep.label,
method: ep.method,
path: ep.path,
description: ep.description,
attributes: ep.attributes.map((attr) => ({
name: attr.name,
type: attr.type,
description: attr.description,
})),
})),
})),
};
}

357
tools/dsl-summary.mjs
View File

@@ -1,357 +0,0 @@
import { readdirSync, readFileSync } from 'node:fs';
import path from 'node:path';
function stripInlineComment(line) {
let inString = false;
let result = '';
for (let index = 0; index < line.length; index += 1) {
const current = line[index];
const next = line[index + 1];
if (current === '"' && line[index - 1] !== '\\') {
inString = !inString;
result += current;
continue;
}
if (!inString && current === '/' && next === '/') {
break;
}
result += current;
}
return result.trim();
}
function parseDefaultValue(rawValue) {
const trimmed = rawValue.trim();
if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
return trimmed.slice(1, -1);
}
if (/^-?\d+$/.test(trimmed)) {
return Number(trimmed);
}
return trimmed;
}
export function getDslFiles(rootDir) {
const domainDir = path.join(rootDir, 'domain');
return readdirSync(domainDir, { withFileTypes: true })
.filter((entry) => entry.isFile() && entry.name.toLowerCase().endsWith('.dsl'))
.map((entry) => path.join(domainDir, entry.name))
.sort((left, right) => left.localeCompare(right));
}
function getOverrideFiles(rootDir) {
const overridesDir = path.join(rootDir, 'overrides');
const files = ['api-overrides.dsl', 'ui-overrides.dsl']
.map((name) => path.join(overridesDir, name))
.filter((filePath) => {
try {
return readdirSync(path.dirname(filePath)).includes(path.basename(filePath));
} catch {
return false;
}
});
return files;
}
export function parseDslFiles(rootDir) {
const dslFiles = getDslFiles(rootDir);
const enums = [];
const entities = [];
const stack = [];
for (const filePath of dslFiles) {
const contents = readFileSync(filePath, 'utf8');
const lines = contents.split(/\r?\n/);
for (const rawLine of lines) {
const line = stripInlineComment(rawLine);
if (!line) {
continue;
}
const top = stack.at(-1);
const enumMatch = line.match(/^enum\s+([A-Za-z][A-Za-z0-9_]*)\s*\{$/);
if (!top && enumMatch) {
const enumDefinition = { name: enumMatch[1], values: [] };
enums.push(enumDefinition);
stack.push({ type: 'enum', ref: enumDefinition });
continue;
}
const entityMatch = line.match(/^entity\s+([A-Za-z][A-Za-z0-9_]*)\s*\{$/);
if (!top && entityMatch) {
const entityDefinition = {
name: entityMatch[1],
primaryKey: null,
fields: [],
foreignKeys: [],
};
entities.push(entityDefinition);
stack.push({ type: 'entity', ref: entityDefinition });
continue;
}
const valueMatch = line.match(/^value\s+([A-Za-z][A-Za-z0-9_]*)\s*\{$/);
if (top?.type === 'enum' && valueMatch) {
const enumValue = { name: valueMatch[1] };
top.ref.values.push(enumValue);
stack.push({ type: 'enumValue', ref: enumValue });
continue;
}
const attributeMatch = line.match(/^attribute\s+([A-Za-z][A-Za-z0-9_]*)\s*\{$/);
if (top?.type === 'entity' && attributeMatch) {
const field = {
name: attributeMatch[1],
type: null,
required: false,
unique: false,
default: null,
};
top.ref.fields.push(field);
stack.push({ type: 'field', ref: field, entity: top.ref });
continue;
}
if (top?.type === 'field' && /^key\s+foreign\s*\{$/.test(line)) {
stack.push({ type: 'foreignKey', ref: top.ref, entity: top.entity });
continue;
}
if (top?.type === 'enumValue') {
const labelMatch = line.match(/^label\s+"(.*)"\s*;$/);
if (labelMatch) {
top.ref.label = labelMatch[1];
continue;
}
}
if (top?.type === 'field') {
const typeMatch = line.match(/^type\s+([A-Za-z][A-Za-z0-9_]*)\s*;$/);
if (typeMatch) {
top.ref.type = typeMatch[1];
continue;
}
if (/^key\s+primary\s*;$/.test(line)) {
top.ref.primary = true;
top.entity.primaryKey = top.ref.name;
continue;
}
const defaultMatch = line.match(/^default\s+(.+)\s*;$/);
if (defaultMatch) {
top.ref.default = parseDefaultValue(defaultMatch[1]);
continue;
}
if (/^is\s+required\s*;$/.test(line)) {
top.ref.required = true;
continue;
}
if (/^is\s+unique\s*;$/.test(line)) {
top.ref.unique = true;
continue;
}
}
if (top?.type === 'foreignKey') {
const relatesMatch = line.match(
/^relates\s+([A-Za-z][A-Za-z0-9_]*)\.([A-Za-z][A-Za-z0-9_]*)\s*;$/,
);
if (relatesMatch) {
const foreignKey = {
field: top.ref.name,
references: {
entity: relatesMatch[1],
field: relatesMatch[2],
},
};
top.ref.foreignKey = foreignKey.references;
top.entity.foreignKeys.push(foreignKey);
continue;
}
}
if (/^}\s*;?$/.test(line)) {
stack.pop();
}
}
}
return { dslFiles, enums, entities };
}
function assertNoDomainRedefinition(line, filePath) {
const blocked = [/^\s*entity\s+/i, /^\s*enum\s+/i, /^\s*attribute\s+/i, /^\s*key\s+primary/i, /^\s*key\s+foreign/i];
for (const pattern of blocked) {
if (pattern.test(line)) {
throw new Error(
`Override file ${path.basename(filePath)} attempts to redefine domain structure: ${line.trim()}`,
);
}
}
}
export function parseOverrides(rootDir, entities) {
const overrideFiles = getOverrideFiles(rootDir);
const entityNames = new Set(entities.map((entity) => entity.name));
const fieldNames = new Set(
entities.flatMap((entity) => entity.fields.map((field) => `${entity.name}.${field.name}`)),
);
const api = { resources: {} };
const ui = { fields: {} };
for (const filePath of overrideFiles) {
const isApi = filePath.endsWith('api-overrides.dsl');
const isUi = filePath.endsWith('ui-overrides.dsl');
const lines = readFileSync(filePath, 'utf8').split(/\r?\n/);
for (const rawLine of lines) {
const line = stripInlineComment(rawLine);
if (!line) {
continue;
}
assertNoDomainRedefinition(line, filePath);
if (isApi) {
const match = line.match(/^resource\s+([A-Za-z][A-Za-z0-9_]*)\s+path\s+"([^"]+)"\s*;$/);
if (!match) {
throw new Error(
`Unsupported API override syntax in ${path.basename(filePath)}: ${line}`,
);
}
const [, entityName, resourcePath] = match;
if (!entityNames.has(entityName)) {
throw new Error(`API override references unknown entity ${entityName}`);
}
api.resources[entityName] = { path: resourcePath };
}
if (isUi) {
const match = line.match(
/^field\s+([A-Za-z][A-Za-z0-9_]*)\.([A-Za-z][A-Za-z0-9_]*)\s+widget\s+"([^"]+)"\s*;$/,
);
if (!match) {
throw new Error(
`Unsupported UI override syntax in ${path.basename(filePath)}: ${line}`,
);
}
const [, entityName, fieldName, widget] = match;
const key = `${entityName}.${fieldName}`;
if (!fieldNames.has(key)) {
throw new Error(`UI override references unknown field ${key}`);
}
ui.fields[key] = { widget };
}
}
}
return { api, ui, sourceFiles: overrideFiles.map((filePath) => path.relative(rootDir, filePath).replaceAll('\\', '/')) };
}
export function buildDomainSummary(rootDir) {
const { dslFiles, enums, entities } = parseDslFiles(rootDir);
parseOverrides(rootDir, entities);
const entityByName = new Map(entities.map((entity) => [entity.name, entity]));
const enumByName = new Map(enums.map((entry) => [entry.name, entry]));
for (const entity of entities) {
if (!entity.primaryKey) {
throw new Error(`Entity ${entity.name} is missing a primary key`);
}
const primaryKeyField = entity.fields.find((field) => field.name === entity.primaryKey);
if (!primaryKeyField) {
throw new Error(
`Entity ${entity.name} declares primary key ${entity.primaryKey}, but the field is missing`,
);
}
if (entity.fields.some((field) => !field.type)) {
throw new Error(`Entity ${entity.name} has attributes with missing type`);
}
for (const foreignKey of entity.foreignKeys) {
const targetEntity = entityByName.get(foreignKey.references.entity);
if (!targetEntity) {
throw new Error(
`Foreign key ${entity.name}.${foreignKey.field} references missing entity ${foreignKey.references.entity}`,
);
}
const targetField = targetEntity.fields.find(
(field) => field.name === foreignKey.references.field,
);
if (!targetField) {
throw new Error(
`Foreign key ${entity.name}.${foreignKey.field} references missing field ${foreignKey.references.entity}.${foreignKey.references.field}`,
);
}
const sourceField = entity.fields.find((field) => field.name === foreignKey.field);
if (sourceField && sourceField.type !== targetField.type) {
throw new Error(
`Foreign key ${entity.name}.${foreignKey.field} type ${sourceField.type} does not match ${foreignKey.references.entity}.${foreignKey.references.field} type ${targetField.type}`,
);
}
}
const fieldNames = new Set();
for (const field of entity.fields) {
if (fieldNames.has(field.name)) {
throw new Error(`Entity ${entity.name} has duplicate field ${field.name}`);
}
fieldNames.add(field.name);
}
}
const entityNames = new Set();
for (const entity of entities) {
if (entityNames.has(entity.name)) {
throw new Error(`Duplicate entity definition: ${entity.name}`);
}
entityNames.add(entity.name);
}
for (const [enumName, enumDefinition] of enumByName.entries()) {
const valueNames = new Set();
for (const value of enumDefinition.values) {
if (valueNames.has(value.name)) {
throw new Error(`Enum ${enumName} has duplicate value ${value.name}`);
}
valueNames.add(value.name);
}
}
return {
sourceFiles: dslFiles.map((filePath) => path.relative(rootDir, filePath).replaceAll('\\', '/')),
entities: entities.map((entity) => ({
name: entity.name,
primaryKey: entity.primaryKey,
fields: entity.fields.map((field) => ({
name: field.name,
type: field.type,
required: field.required,
unique: field.unique,
default: field.default,
})),
foreignKeys: entity.foreignKeys,
})),
enums,
};
}

106
tools/eval/README.md Normal file
View File

@@ -0,0 +1,106 @@
# Eval Harness — Rule 6
Fixture-based regression tests for generated artifacts.
## Why this exists
> "Evals are the test suite for your prompts. You would never ship code without tests;
> don't ship prompts without evals." — Anthropic Engineering
The validation gate (`tools/validate-generation.mjs`) checks **existence** and **structural compliance**.
The eval harness checks **semantic correctness**: are the right patterns present in the generated code?
Do the generated files actually follow the rules in `prompts/`?
Together they enforce:
- Gate: "file exists, field names present, auth seams wired"
- Evals: "DTO has class-validator decorators, FK uses ReferenceInput, date uses DateInput, guard is present"
## Usage
```bash
# Run all evals
npm run eval:generation
# Run evals for one entity
node tools/eval/run-evals.mjs --entity equipment
# Verbose output (show each file being checked)
node tools/eval/run-evals.mjs --verbose
```
## Fixture format
Each fixture lives in `tools/eval/fixtures/<entity>/`:
```
fixtures/
equipment/
meta.json ← what this fixture tests
backend.assertions.json ← patterns the NestJS files must satisfy
frontend.assertions.json ← patterns the React Admin files must satisfy
repair-order/
meta.json
backend.assertions.json
frontend.assertions.json
```
### `meta.json`
```json
{
"entity": "Equipment",
"kebab": "equipment",
"resource": "equipment",
"description": "...",
"tests": ["dto-decorator-coverage", "auth-guards", ...]
}
```
### `*.assertions.json`
Each file entry supports:
| Key | Type | Meaning |
|-----|------|---------|
| `path` | string | Relative path from repo root |
| `must_contain` | string[] | Each string must appear as a literal substring |
| `must_not_contain` | string[] | Each string must NOT appear |
| `must_match_regex` | string[] | Each pattern must match (multiline dot-all) |
| `must_not_match_regex` | string[] | Each pattern must NOT match |
| `comment` | string | Human-readable explanation of what is being tested |
## Eval-driven development workflow
This is the critical principle from Anthropic and Google:
1. **Write the failing eval first.** When you change a prompt or add a rule, add an
assertion that captures the new expectation *before* re-generating.
2. **Run evals**: `npm run eval:generation` → see failures.
3. **Re-generate** the affected entity (following the generation workflow in `AGENTS.md`).
4. **Run evals again**: all pass → the change is verified.
5. **Commit both** the updated fixture and the regenerated artifacts together.
A passing eval after a prompt change confirms the LLM followed the new rule.
A failing eval before a prompt change tells you exactly which prior contract was broken.
## Adding a new entity fixture
When adding a new entity to `domain/toir.api.dsl` and generating its backend + frontend:
1. Create `tools/eval/fixtures/<kebab>/meta.json`
2. Create `tools/eval/fixtures/<kebab>/backend.assertions.json` with at minimum:
- controller: `@Controller(...)`, `@UseGuards(`, `JwtAuthGuard`, HTTP methods
- create_dto: `from 'class-validator'`, required fields with `!:`, `@IsString(`, `@IsOptional(`
- update_dto: `from 'class-validator'`, fields with `?:`, `@IsOptional(`
3. Create `tools/eval/fixtures/<kebab>/frontend.assertions.json` with at minimum:
- create: `ReferenceInput` for FK fields, `NumberInput` for numeric, `DateInput` for date, `SelectInput` for enum
- show: `ReferenceField` for FK fields, `DateField` for date
4. Run `npm run eval:generation` to verify the fixture catches real issues.
## Integration with git hooks
The pre-commit hook (installed by `npm run install-hooks`) runs both:
1. `node tools/validate-generation.mjs --artifacts-only` — existence gate
2. `npm run eval:generation` — semantic eval gate
Both must pass before a commit is accepted.

79
tools/eval/fixtures/equipment/backend.assertions.json Normal file
View File

@@ -0,0 +1,79 @@
{
"entity": "Equipment",
"files": {
"controller": {
"path": "server/src/modules/equipment/equipment.controller.ts",
"must_contain": [
"@Controller('equipments')",
"@UseGuards(",
"JwtAuthGuard",
"@Get()",
"@Post()",
"@Get(':id')",
"@Patch(':id')",
"@Delete(':id')"
],
"must_not_contain": [
"@Put(':id')",
"@Post(':id')"
],
"must_match_regex": [
"@Delete\\(':id'\\)[\\s\\S]{0,80}@Roles\\('admin'\\)|@Roles\\('admin'\\)[\\s\\S]{0,80}@Delete\\(':id'\\)"
],
"comment": "Equipment controller must expose the CRUD verbs expected by the DSL-compatible React Admin contract."
},
"service": {
"path": "server/src/modules/equipment/equipment.service.ts",
"must_contain": [
"setListHeaders(response",
"_start",
"_end",
"_sort",
"_order"
],
"must_match_regex": [
"mode.*insensitive|insensitive.*mode",
"status.*in\\b|\\bin\\b.*status"
],
"comment": "Service must translate React Admin list params into Prisma filters and delegate header wiring through the shared helper."
},
"create_dto": {
"path": "server/src/modules/equipment/dto/create-equipment.dto.ts",
"must_contain": [
"from 'class-validator'",
"inventoryNumber!:",
"name!:",
"equipmentType!:",
"periodicityTO!:",
"status!:",
"@IsString(",
"@IsOptional(",
"@IsEnum("
],
"must_not_contain": [
"id?:",
"id!:"
],
"comment": "Required fields use '!' suffix; optional fields use '?' with @IsOptional(); enum fields use @IsEnum(); class-validator must be imported."
},
"update_dto": {
"path": "server/src/modules/equipment/dto/update-equipment.dto.ts",
"must_contain": [
"from 'class-validator'",
"inventoryNumber?:",
"name?:",
"equipmentType?:",
"status?:",
"@IsOptional(",
"@IsString(",
"@IsEnum("
],
"must_not_contain": [
"inventoryNumber!:",
"name!:",
"status!:"
],
"comment": "Update DTO: all fields are optional ('?' suffix with @IsOptional())."
}
}
}

57
tools/eval/fixtures/equipment/frontend.assertions.json Normal file
View File

@@ -0,0 +1,57 @@
{
"entity": "Equipment",
"resource": "equipment",
"files": {
"list": {
"path": "client/src/resources/equipment/EquipmentList.tsx",
"must_contain": [
"List",
"FilterButton",
"TextField",
"inventoryNumber"
],
"must_match_regex": [
"SelectArrayInput",
"source=\"status\""
],
"comment": "Equipment list must expose filter UI directly and keep enum filters."
},
"create": {
"path": "client/src/resources/equipment/EquipmentCreate.tsx",
"must_contain": [
"Create",
"SimpleForm",
"SelectInput"
],
"must_match_regex": [
"NumberInput[\\s\\S]{0,300}source=\"totalEngineHours\"|source=\"totalEngineHours\"[\\s\\S]{0,300}NumberInput",
"DateInput[\\s\\S]{0,300}source=\"dateOfInspection\"|source=\"dateOfInspection\"[\\s\\S]{0,300}DateInput",
"SelectInput[\\s\\S]{0,300}source=\"status\"|source=\"status\"[\\s\\S]{0,300}SelectInput"
],
"comment": "Equipment create form must keep type-correct inputs for enum, date, and decimal/number fields."
},
"edit": {
"path": "client/src/resources/equipment/EquipmentEdit.tsx",
"must_contain": [
"Edit",
"SimpleForm",
"SelectInput"
],
"must_match_regex": [
"NumberInput[\\s\\S]{0,300}source=\"totalEngineHours\"|source=\"totalEngineHours\"[\\s\\S]{0,300}NumberInput",
"DateInput[\\s\\S]{0,300}source=\"dateOfInspection\"|source=\"dateOfInspection\"[\\s\\S]{0,300}DateInput"
],
"comment": "Equipment edit form must keep the same type-correctness guarantees as create."
},
"show": {
"path": "client/src/resources/equipment/EquipmentShow.tsx",
"must_contain": [
"Show",
"SimpleShowLayout",
"TextField",
"inventoryNumber"
],
"comment": "Show must display key fields including inventoryNumber."
}
}
}

15
tools/eval/fixtures/equipment/meta.json Normal file
View File

@@ -0,0 +1,15 @@
{
"entity": "Equipment",
"kebab": "equipment",
"resource": "equipment",
"description": "Standard entity: UUID primary key, multiple enum fields, decimal fields, date fields, no FK reference to other entities",
"tests": [
"dto-decorator-coverage",
"auth-guards-per-http-method",
"content-range-header-pattern",
"enum-filter-in-operator",
"q-filter-contains-pattern",
"react-admin-component-types",
"class-validator-import"
]
}

62
tools/eval/fixtures/repair-order/backend.assertions.json Normal file
View File

@@ -0,0 +1,62 @@
{
"entity": "CategoryResource",
"files": {
"controller": {
"path": "server/src/modules/category-resource/category-resource.controller.ts",
"must_contain": [
"@Controller('category-resources')",
"@UseGuards(",
"JwtAuthGuard",
"@Get()",
"@Post()",
"@Get(':id')",
"@Patch(':id')",
"@Delete(':id')"
],
"must_not_contain": [
"@Put(':id')"
],
"must_match_regex": [
"@Delete\\(':id'\\)[\\s\\S]{0,120}@Roles\\('admin'\\)|@Roles\\('admin'\\)[\\s\\S]{0,120}@Delete\\(':id'\\)"
]
},
"service": {
"path": "server/src/modules/category-resource/category-resource.service.ts",
"must_contain": [
"setListHeaders",
"_start",
"_end",
"partId",
"employeeCode"
],
"must_match_regex": [
"part:\\s*\\{\\s*is:\\s*\\{\\s*name",
"employee:\\s*\\{\\s*is:\\s*\\{\\s*fullName"
]
},
"create_dto": {
"path": "server/src/modules/category-resource/dto/create-category-resource.dto.ts",
"must_contain": [
"from 'class-validator'",
"partId?:",
"employeeCode?:",
"@IsUUID(",
"@IsString(",
"@IsOptional("
],
"must_not_contain": [
"id?:",
"id!:"
]
},
"update_dto": {
"path": "server/src/modules/category-resource/dto/update-category-resource.dto.ts",
"must_contain": [
"from 'class-validator'",
"@IsOptional(",
"partId?:",
"employeeCode?:"
]
}
}
}

53
tools/eval/fixtures/repair-order/frontend.assertions.json Normal file
View File

@@ -0,0 +1,53 @@
{
"entity": "CategoryResource",
"resource": "category-resources",
"files": {
"list": {
"path": "client/src/resources/category-resource/CategoryResourceList.tsx",
"must_contain": [
"List",
"FilterButton",
"ReferenceField"
],
"must_match_regex": [
"ReferenceField[\\s\\S]{0,200}reference=\"parts\"|reference=\"parts\"[\\s\\S]{0,200}ReferenceField",
"ReferenceField[\\s\\S]{0,200}reference=\"employees\"|reference=\"employees\"[\\s\\S]{0,200}ReferenceField"
]
},
"create": {
"path": "client/src/resources/category-resource/CategoryResourceCreate.tsx",
"must_contain": [
"Create",
"SimpleForm"
],
"must_match_regex": [
"ReferenceInput[\\s\\S]{0,200}reference=\"parts\"|reference=\"parts\"[\\s\\S]{0,200}ReferenceInput",
"ReferenceInput[\\s\\S]{0,200}reference=\"employees\"|reference=\"employees\"[\\s\\S]{0,200}ReferenceInput",
"AutocompleteInput[\\s\\S]{0,200}filterToQuery|filterToQuery[\\s\\S]{0,200}AutocompleteInput"
]
},
"edit": {
"path": "client/src/resources/category-resource/CategoryResourceEdit.tsx",
"must_contain": [
"Edit",
"SimpleForm"
],
"must_match_regex": [
"ReferenceInput[\\s\\S]{0,200}reference=\"parts\"|reference=\"parts\"[\\s\\S]{0,200}ReferenceInput",
"ReferenceInput[\\s\\S]{0,200}reference=\"employees\"|reference=\"employees\"[\\s\\S]{0,200}ReferenceInput"
]
},
"show": {
"path": "client/src/resources/category-resource/CategoryResourceShow.tsx",
"must_contain": [
"Show",
"SimpleShowLayout",
"ReferenceField"
],
"must_match_regex": [
"ReferenceField[\\s\\S]{0,200}reference=\"parts\"|reference=\"parts\"[\\s\\S]{0,200}ReferenceField",
"ReferenceField[\\s\\S]{0,200}reference=\"employees\"|reference=\"employees\"[\\s\\S]{0,200}ReferenceField"
]
}
}
}

13
tools/eval/fixtures/repair-order/meta.json Normal file
View File

@@ -0,0 +1,13 @@
{
"entity": "CategoryResource",
"kebab": "category-resource",
"resource": "category-resources",
"description": "Current FK-heavy entity: UUID PK with references to Part and Employee. Tests reference wiring, autocomplete filters, and protected CRUD routes.",
"tests": [
"dto-decorator-coverage",
"auth-guards",
"fk-reference-input",
"fk-reference-field",
"content-range-header"
]
}

184
tools/eval/run-evals.mjs Normal file
View File

@@ -0,0 +1,184 @@
#!/usr/bin/env node
/**
* tools/eval/run-evals.mjs
*
* Rule 6 — Eval harness: fixture-based regression tests for generated artifacts.
*
* Philosophy:
* - Evals are the test suite for prompts. Never ship a prompt change without
* running evals first.
* - Use deterministic pattern/regex checks ("reference-free" grading) rather
* than golden snapshot comparison. Patterns are maintainable; snapshots are
* brittle.
* - Eval-driven development: write a failing eval FIRST, then update the prompt
* or re-generate to make it pass.
*
* Usage:
* node tools/eval/run-evals.mjs # run all fixtures
* node tools/eval/run-evals.mjs --entity equipment
* node tools/eval/run-evals.mjs --verbose
*/
import { existsSync, readFileSync, readdirSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const rootDir = path.resolve(__dirname, '../..');
const fixturesDir = path.join(__dirname, 'fixtures');
const args = new Set(process.argv.slice(2));
const verbose = args.has('--verbose') || args.has('-v');
const entityFilter = (() => {
const idx = process.argv.indexOf('--entity');
return idx !== -1 ? process.argv[idx + 1] : null;
})();
// ---------------------------------------------------------------------------
// Assertion engine
// ---------------------------------------------------------------------------
let totalChecks = 0;
let totalFailures = 0;
const failures = [];
function readArtifact(relativePath) {
const filePath = path.join(rootDir, relativePath);
if (!existsSync(filePath)) return null;
return readFileSync(filePath, 'utf8');
}
function runFileAssertions(filePath, fileSpec, entityLabel) {
const content = readArtifact(filePath);
if (content === null) {
totalChecks++;
totalFailures++;
failures.push({ entity: entityLabel, file: filePath, check: 'file-exists', result: 'FAIL', detail: `File not found: ${filePath}` });
return;
}
if (verbose) {
console.log(` [${entityLabel}] Checking ${filePath}`);
}
for (const expected of fileSpec.must_contain ?? []) {
totalChecks++;
if (!content.includes(expected)) {
totalFailures++;
failures.push({ entity: entityLabel, file: filePath, check: 'must_contain', result: 'FAIL', detail: `Missing: ${expected}` });
}
}
for (const forbidden of fileSpec.must_not_contain ?? []) {
totalChecks++;
if (content.includes(forbidden)) {
totalFailures++;
failures.push({ entity: entityLabel, file: filePath, check: 'must_not_contain', result: 'FAIL', detail: `Forbidden pattern found: ${forbidden}` });
}
}
for (const patternStr of fileSpec.must_match_regex ?? []) {
totalChecks++;
try {
const re = new RegExp(patternStr);
if (!re.test(content)) {
totalFailures++;
failures.push({ entity: entityLabel, file: filePath, check: 'must_match_regex', result: 'FAIL', detail: `Regex not matched: ${patternStr}` });
}
} catch (e) {
totalFailures++;
failures.push({ entity: entityLabel, file: filePath, check: 'must_match_regex', result: 'ERROR', detail: `Bad regex: ${patternStr} — ${e.message}` });
}
}
for (const patternStr of fileSpec.must_not_match_regex ?? []) {
totalChecks++;
try {
const re = new RegExp(patternStr);
if (re.test(content)) {
totalFailures++;
failures.push({ entity: entityLabel, file: filePath, check: 'must_not_match_regex', result: 'FAIL', detail: `Forbidden regex matched: ${patternStr}` });
}
} catch (e) {
totalFailures++;
failures.push({ entity: entityLabel, file: filePath, check: 'must_not_match_regex', result: 'ERROR', detail: `Bad regex: ${patternStr} — ${e.message}` });
}
}
}
function runFixture(fixtureDir) {
const metaPath = path.join(fixtureDir, 'meta.json');
if (!existsSync(metaPath)) return;
const meta = JSON.parse(readFileSync(metaPath, 'utf8'));
const { entity, kebab } = meta;
if (entityFilter && kebab !== entityFilter && entity.toLowerCase() !== entityFilter.toLowerCase()) {
return;
}
if (verbose) {
console.log(`\n[EVAL] ${entity} — ${meta.description ?? ''}`);
}
const backendPath = path.join(fixtureDir, 'backend.assertions.json');
if (existsSync(backendPath)) {
const spec = JSON.parse(readFileSync(backendPath, 'utf8'));
for (const [key, fileSpec] of Object.entries(spec.files ?? {})) {
runFileAssertions(fileSpec.path, fileSpec, `${entity}/${key}`);
}
}
const frontendPath = path.join(fixtureDir, 'frontend.assertions.json');
if (existsSync(frontendPath)) {
const spec = JSON.parse(readFileSync(frontendPath, 'utf8'));
for (const [key, fileSpec] of Object.entries(spec.files ?? {})) {
runFileAssertions(fileSpec.path, fileSpec, `${entity}/${key}`);
}
}
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
const fixtureDirs = readdirSync(fixturesDir, { withFileTypes: true })
.filter((d) => d.isDirectory())
.map((d) => path.join(fixturesDir, d.name));
for (const dir of fixtureDirs) {
runFixture(dir);
}
// ---------------------------------------------------------------------------
// Report
// ---------------------------------------------------------------------------
console.log('');
console.log('══════════════════════════════════════════════');
console.log(' KIS-TOiR Eval Report');
console.log('══════════════════════════════════════════════');
console.log(` Fixtures: ${fixtureDirs.length}`);
console.log(` Checks: ${totalChecks}`);
console.log(` Passed: ${totalChecks - totalFailures}`);
console.log(` Failed: ${totalFailures}`);
console.log('══════════════════════════════════════════════');
if (failures.length > 0) {
console.log('');
console.log('Failures:');
for (const f of failures) {
console.log(` [${f.result}] ${f.entity} — ${f.file}`);
console.log(` ${f.check}: ${f.detail}`);
}
console.log('');
console.log('To fix: update the prompt or re-generate the failing entity, then re-run evals.');
console.log('To update a fixture (intentional change): edit tools/eval/fixtures/<entity>/*.assertions.json');
console.log('');
process.exit(1);
}
console.log('');
console.log('All evals passed.');
console.log('');

6
tools/generate-domain-summary.mjs → tools/generate-api-summary.mjs
View File

@@ -1,13 +1,13 @@
import { mkdirSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import { buildDomainSummary } from './dsl-summary.mjs';
import { buildApiSummary } from './api-summary.mjs';
const rootDir = process.cwd();
const outputPath = path.join(rootDir, 'domain-summary.json');
const outputPath = path.join(rootDir, 'api-summary.json');
mkdirSync(path.dirname(outputPath), { recursive: true });
const summary = buildDomainSummary(rootDir);
const summary = buildApiSummary(rootDir);
writeFileSync(outputPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');
console.log(`Generated ${path.relative(rootDir, outputPath)}`);

5
tools/hooks/pre-commit Normal file
View File

@@ -0,0 +1,5 @@
#!/bin/sh
# Pre-commit hook: runs the generation validation gate and eval harness.
# Install with: npm run install-hooks
node tools/validate-generation.mjs --artifacts-only && node tools/eval/run-evals.mjs

19
tools/install-hooks.mjs Normal file
View File

@@ -0,0 +1,19 @@
import { copyFileSync, chmodSync, mkdirSync, existsSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const root = path.resolve(__dirname, '..');
const hooksDir = path.join(root, '.git', 'hooks');
const src = path.join(root, 'tools', 'hooks', 'pre-commit');
const dest = path.join(hooksDir, 'pre-commit');
if (!existsSync(path.join(root, '.git'))) {
console.error('Not a git repository. Run from the repo root.');
process.exit(1);
}
mkdirSync(hooksDir, { recursive: true });
copyFileSync(src, dest);
try { chmodSync(dest, 0o755); } catch { /* Windows */ }
console.log('Installed pre-commit hook → .git/hooks/pre-commit');

355
tools/validate-generation.mjs
View File

@@ -1,12 +1,7 @@
import { existsSync, readFileSync, readdirSync } from 'node:fs';
import path from 'node:path';
import { spawnSync } from 'node:child_process';
import {
buildDomainSummary,
getDslFiles,
parseDslFiles,
parseOverrides,
} from './dsl-summary.mjs';
import { getApiDslFiles, parseApiDsl, buildApiSummary } from './api-summary.mjs';
const rootDir = process.cwd();
const args = new Set(process.argv.slice(2));
@@ -125,7 +120,7 @@ function validateBuildChecks() {
'README.md',
'package.json',
'domain/dsl-spec.md',
'domain-summary.json',
'api-summary.json',
'server/prisma/schema.prisma',
'server/.env.example',
'client/.env.example',
@@ -137,23 +132,22 @@ function validateBuildChecks() {
'prompts/validation-rules.md',
]);
const dslFiles = getDslFiles(rootDir).map((filePath) => path.relative(rootDir, filePath).replaceAll('\\', '/'));
assertCondition(dslFiles.length > 0, 'Expected at least one domain/*.dsl file');
// rule: AGENTS.md §Tier-1 — api.dsl must exist
const apiDslFiles = getApiDslFiles(rootDir);
assertCondition(apiDslFiles.length > 0, 'Expected at least one domain/*.api.dsl file');
const actualSummaryRaw = readIfExists('domain-summary.json');
if (actualSummaryRaw) {
const expectedSummary = JSON.stringify(buildDomainSummary(rootDir), null, 2);
assertCondition(
actualSummaryRaw.trim() === expectedSummary,
'domain-summary.json is out of date. Run `npm run generate:domain-summary`.',
);
}
try {
const { entities } = parseDslFiles(rootDir);
parseOverrides(rootDir, entities);
} catch (error) {
failures.push(`Override validation failed: ${error.message}`);
// rule: AGENTS.md §Tier-2 — api-summary.json must match parsed api.dsl
const actualApiSummaryRaw = readIfExists('api-summary.json');
if (actualApiSummaryRaw) {
try {
const expectedApiSummary = JSON.stringify(buildApiSummary(rootDir), null, 2);
assertCondition(
actualApiSummaryRaw.trim() === expectedApiSummary,
'api-summary.json is out of date. Run `npm run generate:api-summary`.',
);
} catch (error) {
failures.push(`api-summary.json freshness check failed: ${error.message}`);
}
}
const { server, client } = getWorkspaceInfo();
@@ -241,12 +235,32 @@ function validateAuthChecks() {
}
function validateNaturalKeyChecks() {
const summary = parseJson('domain-summary.json');
if (!summary) {
// rule: AGENTS.md §Tier-2 — derive natural-key entities from api-summary.json
// A natural-key entity is identified by a root DTO (DTO.X — not Create/Update/Filter/...)
// that has a field annotated with `key primary` where the field name is not 'id'.
const apiSummaryRaw = readIfExists('api-summary.json');
if (!apiSummaryRaw) {
return;
}
const naturalKeyEntities = summary.entities.filter((entity) => entity.primaryKey !== 'id');
let apiSummary;
try {
apiSummary = JSON.parse(apiSummaryRaw);
} catch {
return;
}
const naturalKeyEntities = [];
for (const dto of apiSummary.dtos ?? []) {
// Only root DTOs: DTO.X (not DTO.XCreate / Update / Filter / ListRequest / ListResponse / Page*)
if (/Create$|Update$|Filter$|ListRequest$|ListResponse$|PageRequest$|PageInfo$/.test(dto.name)) continue;
const entityName = dto.name.replace(/^DTO\./, '');
const primaryField = (dto.fields ?? []).find((f) => f.primary === true && f.name !== 'id');
if (primaryField) {
naturalKeyEntities.push({ name: entityName, primaryKey: primaryField.name });
}
}
for (const entity of naturalKeyEntities) {
const moduleName = kebabCase(entity.name);
@@ -349,7 +363,9 @@ function validateRuntimeContractChecks() {
requireFile('docker-compose.yml');
const compose = readIfExists('docker-compose.yml') ?? '';
assertCondition(/image:\s*postgres:16/.test(compose), 'docker-compose must provision postgres:16');
assertCondition(!/keycloak/i.test(compose), 'docker-compose must remain PostgreSQL-only');
const hasKeycloakService =
/^\s{2}keycloak\s*:/m.test(compose) || /image:\s*.*keycloak/i.test(compose);
assertCondition(!hasKeycloakService, 'docker-compose must remain PostgreSQL-only (no Keycloak container)');
const serverEnvExample = readIfExists('server/.env.example') ?? '';
assertCondition(
@@ -482,11 +498,296 @@ function validateRuntimeExecutionChecks() {
runCommand('npx', ['prisma', 'db', 'seed'], serverDir, 'Prisma seed failed');
}
// ---------------------------------------------------------------------------
// Output contract checks — Rule 4 (grounded and schema-bound outputs)
//
// Verify that generated artifacts conform to the output contracts declared in
// prompts/backend-rules.md and prompts/frontend-rules.md.
// All checks are deterministic regex / substring patterns.
// ---------------------------------------------------------------------------
// DSL field type → expected class-validator decorator (pattern fragment).
// rule: backend-rules.md §Type mappings
const DSL_TYPE_TO_CV_DECORATOR = {
uuid: '@IsUUID(',
string: '@IsString(',
text: '@IsString(',
integer: ['@IsInt(', '@IsNumber('],
number: '@IsNumber(',
decimal: '@IsString(',
date: '@IsString(',
boolean: '@IsBoolean(',
};
function escapeRegexStr(s) {
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
// Check that a class-validator decorator appears within 400 chars before fieldName
function fieldHasDecorator(content, fieldName, decoratorFragment) {
const pattern = new RegExp(
`${escapeRegexStr(decoratorFragment)}[\\s\\S]{0,400}${escapeRegexStr(fieldName)}[?!]?\\s*:`,
);
return pattern.test(content);
}
function validateDtoDecoratorCoverage() {
const { dtos, enums } = parseApiDsl(rootDir);
const enumNames = new Set(enums.map((e) => e.name));
for (const dto of dtos) {
const createMatch = dto.name.match(/^DTO\.(\w+)Create$/);
const updateMatch = dto.name.match(/^DTO\.(\w+)Update$/);
const match = createMatch ?? updateMatch;
if (!match) continue;
const kebab = kebabCase(match[1]);
const prefix = createMatch ? 'create' : 'update';
const dtoPath = `server/src/modules/${kebab}/dto/${prefix}-${kebab}.dto.ts`;
const content = readIfExists(dtoPath) ?? '';
if (!content) continue;
// rule: backend-rules.md §Type mappings — every DTO must import class-validator
assertCondition(
/from 'class-validator'/.test(content),
`${dtoPath}: missing import from 'class-validator' — rule: backend-rules.md §Type mappings`,
);
for (const field of dto.fields) {
const { name, type, nullable, required } = field;
if (!type) continue;
// Skip DTO reference types — validated by @ValidateNested separately
if (type.startsWith('DTO.')) continue;
// rule: backend-rules.md — nullable/optional fields must carry @IsOptional()
if (!required || nullable) {
assertCondition(
fieldHasDecorator(content, name, '@IsOptional('),
`${dtoPath}: field '${name}' is optional/nullable but missing @IsOptional()`,
);
}
// rule: backend-rules.md §Type mappings — type-correct decorator
const bareType = type.replace('[]', '');
if (enumNames.has(bareType)) {
assertCondition(
fieldHasDecorator(content, name, `@IsEnum(${bareType}`),
`${dtoPath}: field '${name}' has enum type '${bareType}' but missing @IsEnum(${bareType})`,
);
} else {
const expected = DSL_TYPE_TO_CV_DECORATOR[bareType];
if (expected) {
const options = Array.isArray(expected) ? expected : [expected];
const found = options.some((opt) => fieldHasDecorator(content, name, opt));
assertCondition(
found,
`${dtoPath}: field '${name}' has type '${bareType}' but missing ${options.join(' or ')}`,
);
}
}
}
}
}
function validateControllerGuards() {
// rule: backend-rules.md §Backend auth defaults — every controller needs JwtAuthGuard
const { apis } = parseApiDsl(rootDir);
for (const api of apis) {
const resourceName = api.name.replace(/^API\./, '');
const kebab = kebabCase(resourceName);
const controllerPath = `server/src/modules/${kebab}/${kebab}.controller.ts`;
const content = readIfExists(controllerPath) ?? '';
if (!content) continue;
// UseGuards must appear at the class level (within first 800 chars before the class declaration)
assertCondition(
/@UseGuards\s*\(/.test(content),
`${controllerPath}: missing @UseGuards(...) — all controllers must guard their routes`,
);
// JwtAuthGuard or equivalent JWT guard must be referenced
assertCondition(
/JwtAuthGuard|JwtGuard|AuthGuard/.test(content),
`${controllerPath}: controller must use JwtAuthGuard (or equivalent) — rule: backend-rules.md §Backend auth defaults`,
);
// rule: backend-rules.md §Backend auth defaults — DELETE must be admin-only
if (api.endpoints.some((ep) => ep.method === 'DELETE')) {
assertCondition(
/@Roles\s*\([^)]*'admin'/.test(content) || /@Roles\s*\([^)]*"admin"/.test(content),
`${controllerPath}: DELETE endpoints require @Roles('admin') — rule: backend-rules.md §Backend auth defaults`,
);
}
}
}
function validateFrontendComponentTypes() {
// rule: frontend-rules.md §Resource generation — type-safe component mapping
const { dtos, enums } = parseApiDsl(rootDir);
const enumNames = new Set(enums.map((e) => e.name));
for (const dto of dtos) {
const createMatch = dto.name.match(/^DTO\.(\w+)Create$/);
if (!createMatch) continue;
const resourceName = createMatch[1];
const kebab = kebabCase(resourceName);
const createPath = `client/src/resources/${kebab}/${resourceName}Create.tsx`;
const editPath = `client/src/resources/${kebab}/${resourceName}Edit.tsx`;
for (const componentPath of [createPath, editPath]) {
const content = readIfExists(componentPath) ?? '';
if (!content) continue;
for (const field of dto.fields) {
const { name, type } = field;
if (!type) continue;
const bareType = type.replace('[]', '');
if (bareType === 'integer' || bareType === 'number' || bareType === 'decimal') {
// rule: frontend-rules.md — integer/number/decimal → NumberInput, never TextInput
const usesNumberInput = content.includes(`source="${name}"`) &&
new RegExp(`NumberInput[^>]*source="${escapeRegexStr(name)}"|source="${escapeRegexStr(name)}"[^>]*NumberInput`).test(content);
// Only flag if TextInput is clearly used for a numeric field
const usesTextForNumeric = new RegExp(
`TextInput[\\s\\S]{0,200}source="${escapeRegexStr(name)}"`,
).test(content);
assertCondition(
!usesTextForNumeric,
`${componentPath}: field '${name}' has type '${bareType}' but uses TextInput — must use NumberInput`,
);
}
if (bareType === 'date') {
const usesTextForDate = new RegExp(
`TextInput[\\s\\S]{0,200}source="${escapeRegexStr(name)}"`,
).test(content);
assertCondition(
!usesTextForDate,
`${componentPath}: field '${name}' has type 'date' but uses TextInput — must use DateInput`,
);
}
}
}
}
}
// ---------------------------------------------------------------------------
// api.dsl coverage checks
//
// The API DSL is parsed by tools/api-summary.mjs — the single canonical
// parser. This section contains only mechanical gate logic; no DSL parsing
// or generation semantics live here.
// ---------------------------------------------------------------------------
function validateApiDslCoverage() {
const apiDslFiles = getApiDslFiles(rootDir);
if (apiDslFiles.length === 0) {
warn('No domain/*.api.dsl files found. Skipping api.dsl coverage checks.');
return;
}
// rule: AGENTS.md §Tier-3 generation zones, backend-rules.md §api-dsl-as-source
const { apis, dtos } = parseApiDsl(rootDir);
for (const api of apis) {
// api.name is "API.Equipment"; derive the kebab resource name
const resourceName = api.name.replace(/^API\./, '');
const kebab = kebabCase(resourceName);
// rule: backend-rules.md §module-file-structure
requireFiles([
`server/src/modules/${kebab}/${kebab}.module.ts`,
`server/src/modules/${kebab}/${kebab}.controller.ts`,
`server/src/modules/${kebab}/${kebab}.service.ts`,
`server/src/modules/${kebab}/dto/create-${kebab}.dto.ts`,
`server/src/modules/${kebab}/dto/update-${kebab}.dto.ts`,
]);
// rule: frontend-rules.md §resource-file-structure
requireFiles([
`client/src/resources/${kebab}/${resourceName}List.tsx`,
`client/src/resources/${kebab}/${resourceName}Create.tsx`,
`client/src/resources/${kebab}/${resourceName}Edit.tsx`,
`client/src/resources/${kebab}/${resourceName}Show.tsx`,
]);
const controllerContent =
readIfExists(`server/src/modules/${kebab}/${kebab}.controller.ts`) ?? '';
if (!controllerContent) continue;
// rule: backend-rules.md §endpoint-http-method-mapping
for (const ep of api.endpoints) {
if (!ep.label) continue;
const isPageEndpoint = (ep.path ?? '').endsWith('/page');
let found = false;
if (ep.method === 'GET') {
found = controllerContent.includes('@Get(');
} else if (ep.method === 'POST' && isPageEndpoint) {
found = controllerContent.includes('@Post(') || controllerContent.includes('@Get(');
} else if (ep.method === 'POST') {
found = controllerContent.includes('@Post(');
} else if (ep.method === 'PUT') {
found = controllerContent.includes('@Put(') || controllerContent.includes('@Patch(');
} else if (ep.method === 'DELETE') {
found = controllerContent.includes('@Delete(');
}
assertCondition(
found,
`${api.name} endpoint ${ep.name} (${ep.label}): no matching HTTP handler in controller`,
);
}
}
// rule: backend-rules.md §DTO-field-coverage
for (const dto of dtos) {
const createMatch = dto.name.match(/^DTO\.(\w+)Create$/);
const updateMatch = dto.name.match(/^DTO\.(\w+)Update$/);
if (createMatch) {
const kebab = kebabCase(createMatch[1]);
const dtoPath = `server/src/modules/${kebab}/dto/create-${kebab}.dto.ts`;
const content = readIfExists(dtoPath) ?? '';
if (content) {
for (const field of dto.fields) {
assertCondition(
content.includes(field.name),
`${dto.name} field '${field.name}' missing from ${dtoPath}`,
);
}
}
}
if (updateMatch) {
const kebab = kebabCase(updateMatch[1]);
const dtoPath = `server/src/modules/${kebab}/dto/update-${kebab}.dto.ts`;
const content = readIfExists(dtoPath) ?? '';
if (content) {
for (const field of dto.fields) {
assertCondition(
content.includes(field.name),
`${dto.name} field '${field.name}' missing from ${dtoPath}`,
);
}
}
}
}
}
// ---------------------------------------------------------------------------
validateBuildChecks();
validateAuthChecks();
validateNaturalKeyChecks();
validateRealmChecks();
validateRuntimeContractChecks();
validateApiDslCoverage();
validateDtoDecoratorCoverage();
validateControllerGuards();
validateFrontendComponentTypes();
if (!artifactsOnly) {
validateBuildExecutionChecks();