(llm-first): context budget, validation, and eval harness, orchestration general-prompt
This commit is contained in:
MaKarin
@@ -1,12 +1,7 @@
|
||||
import { existsSync, readFileSync, readdirSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import {
|
||||
buildDomainSummary,
|
||||
getDslFiles,
|
||||
parseDslFiles,
|
||||
parseOverrides,
|
||||
} from './dsl-summary.mjs';
|
||||
import { getApiDslFiles, parseApiDsl, buildApiSummary } from './api-summary.mjs';
|
||||
|
||||
const rootDir = process.cwd();
|
||||
const args = new Set(process.argv.slice(2));
|
||||
@@ -125,7 +120,7 @@ function validateBuildChecks() {
|
||||
'README.md',
|
||||
'package.json',
|
||||
'domain/dsl-spec.md',
|
||||
'domain-summary.json',
|
||||
'api-summary.json',
|
||||
'server/prisma/schema.prisma',
|
||||
'server/.env.example',
|
||||
'client/.env.example',
|
||||
@@ -137,23 +132,22 @@ function validateBuildChecks() {
|
||||
'prompts/validation-rules.md',
|
||||
]);
|
||||
|
||||
const dslFiles = getDslFiles(rootDir).map((filePath) => path.relative(rootDir, filePath).replaceAll('\\', '/'));
|
||||
assertCondition(dslFiles.length > 0, 'Expected at least one domain/*.dsl file');
|
||||
// rule: AGENTS.md §Tier-1 — api.dsl must exist
|
||||
const apiDslFiles = getApiDslFiles(rootDir);
|
||||
assertCondition(apiDslFiles.length > 0, 'Expected at least one domain/*.api.dsl file');
|
||||
|
||||
const actualSummaryRaw = readIfExists('domain-summary.json');
|
||||
if (actualSummaryRaw) {
|
||||
const expectedSummary = JSON.stringify(buildDomainSummary(rootDir), null, 2);
|
||||
assertCondition(
|
||||
actualSummaryRaw.trim() === expectedSummary,
|
||||
'domain-summary.json is out of date. Run `npm run generate:domain-summary`.',
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const { entities } = parseDslFiles(rootDir);
|
||||
parseOverrides(rootDir, entities);
|
||||
} catch (error) {
|
||||
failures.push(`Override validation failed: ${error.message}`);
|
||||
// rule: AGENTS.md §Tier-2 — api-summary.json must match parsed api.dsl
|
||||
const actualApiSummaryRaw = readIfExists('api-summary.json');
|
||||
if (actualApiSummaryRaw) {
|
||||
try {
|
||||
const expectedApiSummary = JSON.stringify(buildApiSummary(rootDir), null, 2);
|
||||
assertCondition(
|
||||
actualApiSummaryRaw.trim() === expectedApiSummary,
|
||||
'api-summary.json is out of date. Run `npm run generate:api-summary`.',
|
||||
);
|
||||
} catch (error) {
|
||||
failures.push(`api-summary.json freshness check failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const { server, client } = getWorkspaceInfo();
|
||||
@@ -241,12 +235,32 @@ function validateAuthChecks() {
|
||||
}
|
||||
|
||||
function validateNaturalKeyChecks() {
|
||||
const summary = parseJson('domain-summary.json');
|
||||
if (!summary) {
|
||||
// rule: AGENTS.md §Tier-2 — derive natural-key entities from api-summary.json
|
||||
// A natural-key entity is identified by a root DTO (DTO.X — not Create/Update/Filter/...)
|
||||
// that has a field annotated with `key primary` where the field name is not 'id'.
|
||||
const apiSummaryRaw = readIfExists('api-summary.json');
|
||||
if (!apiSummaryRaw) {
|
||||
return;
|
||||
}
|
||||
|
||||
const naturalKeyEntities = summary.entities.filter((entity) => entity.primaryKey !== 'id');
|
||||
let apiSummary;
|
||||
try {
|
||||
apiSummary = JSON.parse(apiSummaryRaw);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
const naturalKeyEntities = [];
|
||||
for (const dto of apiSummary.dtos ?? []) {
|
||||
// Only root DTOs: DTO.X (not DTO.XCreate / Update / Filter / ListRequest / ListResponse / Page*)
|
||||
if (/Create$|Update$|Filter$|ListRequest$|ListResponse$|PageRequest$|PageInfo$/.test(dto.name)) continue;
|
||||
|
||||
const entityName = dto.name.replace(/^DTO\./, '');
|
||||
const primaryField = (dto.fields ?? []).find((f) => f.primary === true && f.name !== 'id');
|
||||
if (primaryField) {
|
||||
naturalKeyEntities.push({ name: entityName, primaryKey: primaryField.name });
|
||||
}
|
||||
}
|
||||
|
||||
for (const entity of naturalKeyEntities) {
|
||||
const moduleName = kebabCase(entity.name);
|
||||
@@ -349,7 +363,9 @@ function validateRuntimeContractChecks() {
|
||||
requireFile('docker-compose.yml');
|
||||
const compose = readIfExists('docker-compose.yml') ?? '';
|
||||
assertCondition(/image:\s*postgres:16/.test(compose), 'docker-compose must provision postgres:16');
|
||||
assertCondition(!/keycloak/i.test(compose), 'docker-compose must remain PostgreSQL-only');
|
||||
const hasKeycloakService =
|
||||
/^\s{2}keycloak\s*:/m.test(compose) || /image:\s*.*keycloak/i.test(compose);
|
||||
assertCondition(!hasKeycloakService, 'docker-compose must remain PostgreSQL-only (no Keycloak container)');
|
||||
|
||||
const serverEnvExample = readIfExists('server/.env.example') ?? '';
|
||||
assertCondition(
|
||||
@@ -482,11 +498,296 @@ function validateRuntimeExecutionChecks() {
|
||||
runCommand('npx', ['prisma', 'db', 'seed'], serverDir, 'Prisma seed failed');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Output contract checks — Rule 4 (grounded and schema-bound outputs)
|
||||
//
|
||||
// Verify that generated artifacts conform to the output contracts declared in
|
||||
// prompts/backend-rules.md and prompts/frontend-rules.md.
|
||||
// All checks are deterministic regex / substring patterns.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// DSL field type → expected class-validator decorator (pattern fragment).
|
||||
// rule: backend-rules.md §Type mappings
|
||||
const DSL_TYPE_TO_CV_DECORATOR = {
|
||||
uuid: '@IsUUID(',
|
||||
string: '@IsString(',
|
||||
text: '@IsString(',
|
||||
integer: ['@IsInt(', '@IsNumber('],
|
||||
number: '@IsNumber(',
|
||||
decimal: '@IsString(',
|
||||
date: '@IsString(',
|
||||
boolean: '@IsBoolean(',
|
||||
};
|
||||
|
||||
function escapeRegexStr(s) {
|
||||
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
// Check that a class-validator decorator appears within 400 chars before fieldName
|
||||
function fieldHasDecorator(content, fieldName, decoratorFragment) {
|
||||
const pattern = new RegExp(
|
||||
`${escapeRegexStr(decoratorFragment)}[\\s\\S]{0,400}${escapeRegexStr(fieldName)}[?!]?\\s*:`,
|
||||
);
|
||||
return pattern.test(content);
|
||||
}
|
||||
|
||||
function validateDtoDecoratorCoverage() {
|
||||
const { dtos, enums } = parseApiDsl(rootDir);
|
||||
const enumNames = new Set(enums.map((e) => e.name));
|
||||
|
||||
for (const dto of dtos) {
|
||||
const createMatch = dto.name.match(/^DTO\.(\w+)Create$/);
|
||||
const updateMatch = dto.name.match(/^DTO\.(\w+)Update$/);
|
||||
const match = createMatch ?? updateMatch;
|
||||
if (!match) continue;
|
||||
|
||||
const kebab = kebabCase(match[1]);
|
||||
const prefix = createMatch ? 'create' : 'update';
|
||||
const dtoPath = `server/src/modules/${kebab}/dto/${prefix}-${kebab}.dto.ts`;
|
||||
const content = readIfExists(dtoPath) ?? '';
|
||||
if (!content) continue;
|
||||
|
||||
// rule: backend-rules.md §Type mappings — every DTO must import class-validator
|
||||
assertCondition(
|
||||
/from 'class-validator'/.test(content),
|
||||
`${dtoPath}: missing import from 'class-validator' — rule: backend-rules.md §Type mappings`,
|
||||
);
|
||||
|
||||
for (const field of dto.fields) {
|
||||
const { name, type, nullable, required } = field;
|
||||
if (!type) continue;
|
||||
|
||||
// Skip DTO reference types — validated by @ValidateNested separately
|
||||
if (type.startsWith('DTO.')) continue;
|
||||
|
||||
// rule: backend-rules.md — nullable/optional fields must carry @IsOptional()
|
||||
if (!required || nullable) {
|
||||
assertCondition(
|
||||
fieldHasDecorator(content, name, '@IsOptional('),
|
||||
`${dtoPath}: field '${name}' is optional/nullable but missing @IsOptional()`,
|
||||
);
|
||||
}
|
||||
|
||||
// rule: backend-rules.md §Type mappings — type-correct decorator
|
||||
const bareType = type.replace('[]', '');
|
||||
if (enumNames.has(bareType)) {
|
||||
assertCondition(
|
||||
fieldHasDecorator(content, name, `@IsEnum(${bareType}`),
|
||||
`${dtoPath}: field '${name}' has enum type '${bareType}' but missing @IsEnum(${bareType})`,
|
||||
);
|
||||
} else {
|
||||
const expected = DSL_TYPE_TO_CV_DECORATOR[bareType];
|
||||
if (expected) {
|
||||
const options = Array.isArray(expected) ? expected : [expected];
|
||||
const found = options.some((opt) => fieldHasDecorator(content, name, opt));
|
||||
assertCondition(
|
||||
found,
|
||||
`${dtoPath}: field '${name}' has type '${bareType}' but missing ${options.join(' or ')}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function validateControllerGuards() {
|
||||
// rule: backend-rules.md §Backend auth defaults — every controller needs JwtAuthGuard
|
||||
const { apis } = parseApiDsl(rootDir);
|
||||
for (const api of apis) {
|
||||
const resourceName = api.name.replace(/^API\./, '');
|
||||
const kebab = kebabCase(resourceName);
|
||||
const controllerPath = `server/src/modules/${kebab}/${kebab}.controller.ts`;
|
||||
const content = readIfExists(controllerPath) ?? '';
|
||||
if (!content) continue;
|
||||
|
||||
// UseGuards must appear at the class level (within first 800 chars before the class declaration)
|
||||
assertCondition(
|
||||
/@UseGuards\s*\(/.test(content),
|
||||
`${controllerPath}: missing @UseGuards(...) — all controllers must guard their routes`,
|
||||
);
|
||||
|
||||
// JwtAuthGuard or equivalent JWT guard must be referenced
|
||||
assertCondition(
|
||||
/JwtAuthGuard|JwtGuard|AuthGuard/.test(content),
|
||||
`${controllerPath}: controller must use JwtAuthGuard (or equivalent) — rule: backend-rules.md §Backend auth defaults`,
|
||||
);
|
||||
|
||||
// rule: backend-rules.md §Backend auth defaults — DELETE must be admin-only
|
||||
if (api.endpoints.some((ep) => ep.method === 'DELETE')) {
|
||||
assertCondition(
|
||||
/@Roles\s*\([^)]*'admin'/.test(content) || /@Roles\s*\([^)]*"admin"/.test(content),
|
||||
`${controllerPath}: DELETE endpoints require @Roles('admin') — rule: backend-rules.md §Backend auth defaults`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function validateFrontendComponentTypes() {
|
||||
// rule: frontend-rules.md §Resource generation — type-safe component mapping
|
||||
const { dtos, enums } = parseApiDsl(rootDir);
|
||||
const enumNames = new Set(enums.map((e) => e.name));
|
||||
|
||||
for (const dto of dtos) {
|
||||
const createMatch = dto.name.match(/^DTO\.(\w+)Create$/);
|
||||
if (!createMatch) continue;
|
||||
|
||||
const resourceName = createMatch[1];
|
||||
const kebab = kebabCase(resourceName);
|
||||
const createPath = `client/src/resources/${kebab}/${resourceName}Create.tsx`;
|
||||
const editPath = `client/src/resources/${kebab}/${resourceName}Edit.tsx`;
|
||||
|
||||
for (const componentPath of [createPath, editPath]) {
|
||||
const content = readIfExists(componentPath) ?? '';
|
||||
if (!content) continue;
|
||||
|
||||
for (const field of dto.fields) {
|
||||
const { name, type } = field;
|
||||
if (!type) continue;
|
||||
const bareType = type.replace('[]', '');
|
||||
|
||||
if (bareType === 'integer' || bareType === 'number' || bareType === 'decimal') {
|
||||
// rule: frontend-rules.md — integer/number/decimal → NumberInput, never TextInput
|
||||
const usesNumberInput = content.includes(`source="${name}"`) &&
|
||||
new RegExp(`NumberInput[^>]*source="${escapeRegexStr(name)}"|source="${escapeRegexStr(name)}"[^>]*NumberInput`).test(content);
|
||||
// Only flag if TextInput is clearly used for a numeric field
|
||||
const usesTextForNumeric = new RegExp(
|
||||
`TextInput[\\s\\S]{0,200}source="${escapeRegexStr(name)}"`,
|
||||
).test(content);
|
||||
assertCondition(
|
||||
!usesTextForNumeric,
|
||||
`${componentPath}: field '${name}' has type '${bareType}' but uses TextInput — must use NumberInput`,
|
||||
);
|
||||
}
|
||||
|
||||
if (bareType === 'date') {
|
||||
const usesTextForDate = new RegExp(
|
||||
`TextInput[\\s\\S]{0,200}source="${escapeRegexStr(name)}"`,
|
||||
).test(content);
|
||||
assertCondition(
|
||||
!usesTextForDate,
|
||||
`${componentPath}: field '${name}' has type 'date' but uses TextInput — must use DateInput`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// api.dsl coverage checks
|
||||
//
|
||||
// The API DSL is parsed by tools/api-summary.mjs — the single canonical
|
||||
// parser. This section contains only mechanical gate logic; no DSL parsing
|
||||
// or generation semantics live here.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function validateApiDslCoverage() {
|
||||
const apiDslFiles = getApiDslFiles(rootDir);
|
||||
if (apiDslFiles.length === 0) {
|
||||
warn('No domain/*.api.dsl files found. Skipping api.dsl coverage checks.');
|
||||
return;
|
||||
}
|
||||
|
||||
// rule: AGENTS.md §Tier-3 generation zones, backend-rules.md §api-dsl-as-source
|
||||
const { apis, dtos } = parseApiDsl(rootDir);
|
||||
|
||||
for (const api of apis) {
|
||||
// api.name is "API.Equipment"; derive the kebab resource name
|
||||
const resourceName = api.name.replace(/^API\./, '');
|
||||
const kebab = kebabCase(resourceName);
|
||||
|
||||
// rule: backend-rules.md §module-file-structure
|
||||
requireFiles([
|
||||
`server/src/modules/${kebab}/${kebab}.module.ts`,
|
||||
`server/src/modules/${kebab}/${kebab}.controller.ts`,
|
||||
`server/src/modules/${kebab}/${kebab}.service.ts`,
|
||||
`server/src/modules/${kebab}/dto/create-${kebab}.dto.ts`,
|
||||
`server/src/modules/${kebab}/dto/update-${kebab}.dto.ts`,
|
||||
]);
|
||||
|
||||
// rule: frontend-rules.md §resource-file-structure
|
||||
requireFiles([
|
||||
`client/src/resources/${kebab}/${resourceName}List.tsx`,
|
||||
`client/src/resources/${kebab}/${resourceName}Create.tsx`,
|
||||
`client/src/resources/${kebab}/${resourceName}Edit.tsx`,
|
||||
`client/src/resources/${kebab}/${resourceName}Show.tsx`,
|
||||
]);
|
||||
|
||||
const controllerContent =
|
||||
readIfExists(`server/src/modules/${kebab}/${kebab}.controller.ts`) ?? '';
|
||||
if (!controllerContent) continue;
|
||||
|
||||
// rule: backend-rules.md §endpoint-http-method-mapping
|
||||
for (const ep of api.endpoints) {
|
||||
if (!ep.label) continue;
|
||||
const isPageEndpoint = (ep.path ?? '').endsWith('/page');
|
||||
|
||||
let found = false;
|
||||
if (ep.method === 'GET') {
|
||||
found = controllerContent.includes('@Get(');
|
||||
} else if (ep.method === 'POST' && isPageEndpoint) {
|
||||
found = controllerContent.includes('@Post(') || controllerContent.includes('@Get(');
|
||||
} else if (ep.method === 'POST') {
|
||||
found = controllerContent.includes('@Post(');
|
||||
} else if (ep.method === 'PUT') {
|
||||
found = controllerContent.includes('@Put(') || controllerContent.includes('@Patch(');
|
||||
} else if (ep.method === 'DELETE') {
|
||||
found = controllerContent.includes('@Delete(');
|
||||
}
|
||||
|
||||
assertCondition(
|
||||
found,
|
||||
`${api.name} endpoint ${ep.name} (${ep.label}): no matching HTTP handler in controller`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// rule: backend-rules.md §DTO-field-coverage
|
||||
for (const dto of dtos) {
|
||||
const createMatch = dto.name.match(/^DTO\.(\w+)Create$/);
|
||||
const updateMatch = dto.name.match(/^DTO\.(\w+)Update$/);
|
||||
|
||||
if (createMatch) {
|
||||
const kebab = kebabCase(createMatch[1]);
|
||||
const dtoPath = `server/src/modules/${kebab}/dto/create-${kebab}.dto.ts`;
|
||||
const content = readIfExists(dtoPath) ?? '';
|
||||
if (content) {
|
||||
for (const field of dto.fields) {
|
||||
assertCondition(
|
||||
content.includes(field.name),
|
||||
`${dto.name} field '${field.name}' missing from ${dtoPath}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (updateMatch) {
|
||||
const kebab = kebabCase(updateMatch[1]);
|
||||
const dtoPath = `server/src/modules/${kebab}/dto/update-${kebab}.dto.ts`;
|
||||
const content = readIfExists(dtoPath) ?? '';
|
||||
if (content) {
|
||||
for (const field of dto.fields) {
|
||||
assertCondition(
|
||||
content.includes(field.name),
|
||||
`${dto.name} field '${field.name}' missing from ${dtoPath}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
validateBuildChecks();
|
||||
validateAuthChecks();
|
||||
validateNaturalKeyChecks();
|
||||
validateRealmChecks();
|
||||
validateRuntimeContractChecks();
|
||||
validateApiDslCoverage();
|
||||
validateDtoDecoratorCoverage();
|
||||
validateControllerGuards();
|
||||
validateFrontendComponentTypes();
|
||||
|
||||
if (!artifactsOnly) {
|
||||
validateBuildExecutionChecks();
|
||||
|
||||
Reference in New Issue
Block a user