Убран лишний код. Скорректированы docker и package файлы. Переработано взаимодействие с бд - упрощена логика.

This commit is contained in:
Первов Артем
2026-06-19 14:59:39 +03:00
parent f5ed7fa9be
commit a12d490513
31 changed files with 120 additions and 552 deletions

4
.dockerignore Normal file
View File

@@ -0,0 +1,4 @@
node_modules
dist
.env
.env.*

View File

@@ -13,7 +13,3 @@ INGEST_MAX_BATCH_SIZE=10000
# Poll interval for GET /current/events SSE snapshots. # Poll interval for GET /current/events SSE snapshots.
CURRENT_EVENTS_POLL_MS=1000 CURRENT_EVENTS_POLL_MS=1000
# Graph API defaults.
HISTORY_TARGET_POINTS=2000
HISTORY_MAX_TARGET_POINTS=5000

View File

@@ -5,5 +5,3 @@ INGEST_API_KEY=
PG_POOL_MAX=20 PG_POOL_MAX=20
INGEST_MAX_BATCH_SIZE=10000 INGEST_MAX_BATCH_SIZE=10000
CURRENT_EVENTS_POLL_MS=1000 CURRENT_EVENTS_POLL_MS=1000
HISTORY_TARGET_POINTS=2000
HISTORY_MAX_TARGET_POINTS=5000

View File

@@ -1,18 +1,14 @@
FROM node:22-alpine AS deps FROM node:22-alpine
WORKDIR /app WORKDIR /app
COPY package*.json ./ COPY package*.json ./
RUN npm ci RUN npm ci
FROM node:22-alpine AS build COPY tsconfig*.json ./
WORKDIR /app COPY src ./src
COPY --from=deps /app/node_modules ./node_modules
COPY . . RUN npm run build && npm prune --omit=dev
RUN npm run build
FROM node:22-alpine AS runtime
WORKDIR /app
ENV NODE_ENV=production ENV NODE_ENV=production
COPY package*.json ./
RUN npm ci --omit=dev
COPY --from=build /app/dist ./dist
CMD ["node", "dist/main.js"] CMD ["node", "dist/main.js"]

View File

@@ -5,17 +5,15 @@ services:
restart: unless-stopped restart: unless-stopped
environment: environment:
NODE_ENV: production NODE_ENV: production
PORT: ${PORT:-3101} PORT: ${PORT}
DATABASE_URL: ${DATABASE_URL} DATABASE_URL: ${DATABASE_URL}
CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:-} CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS}
INGEST_API_KEY: ${INGEST_API_KEY:-} INGEST_API_KEY: ${INGEST_API_KEY}
PG_POOL_MAX: ${PG_POOL_MAX:-20} PG_POOL_MAX: ${PG_POOL_MAX}
INGEST_MAX_BATCH_SIZE: ${INGEST_MAX_BATCH_SIZE:-10000} INGEST_MAX_BATCH_SIZE: ${INGEST_MAX_BATCH_SIZE}
CURRENT_EVENTS_POLL_MS: ${CURRENT_EVENTS_POLL_MS:-1000} CURRENT_EVENTS_POLL_MS: ${CURRENT_EVENTS_POLL_MS}
HISTORY_TARGET_POINTS: ${HISTORY_TARGET_POINTS:-2000}
HISTORY_MAX_TARGET_POINTS: ${HISTORY_MAX_TARGET_POINTS:-5000}
ports: ports:
- "${PORT:-3101}:${PORT:-3101}" - "${PORT}:${PORT}"
networks: networks:
- proxy - proxy

56
package-lock.json generated
View File

@@ -10,7 +10,6 @@
"license": "PERSONAL", "license": "PERSONAL",
"dependencies": { "dependencies": {
"@nestjs/common": "^11.0.1", "@nestjs/common": "^11.0.1",
"@nestjs/config": "^4.0.2",
"@nestjs/core": "^11.0.1", "@nestjs/core": "^11.0.1",
"@nestjs/platform-express": "^11.0.1", "@nestjs/platform-express": "^11.0.1",
"class-transformer": "^0.5.1", "class-transformer": "^0.5.1",
@@ -2359,21 +2358,6 @@
} }
} }
}, },
"node_modules/@nestjs/config": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/@nestjs/config/-/config-4.0.4.tgz",
"integrity": "sha512-CJPjNitr0bAufSEnRe2N+JbnVmMmDoo6hvKCPzXgZoGwJSmp/dZPk9f/RMbuD/+Q1ZJPjwsRpq0vxna++Knwow==",
"license": "MIT",
"dependencies": {
"dotenv": "17.4.1",
"dotenv-expand": "12.0.3",
"lodash": "4.18.1"
},
"peerDependencies": {
"@nestjs/common": "^10.0.0 || ^11.0.0",
"rxjs": "^7.1.0"
}
},
"node_modules/@nestjs/core": { "node_modules/@nestjs/core": {
"version": "11.1.26", "version": "11.1.26",
"resolved": "https://registry.npmjs.org/@nestjs/core/-/core-11.1.26.tgz", "resolved": "https://registry.npmjs.org/@nestjs/core/-/core-11.1.26.tgz",
@@ -4938,45 +4922,6 @@
"node": ">=0.3.1" "node": ">=0.3.1"
} }
}, },
"node_modules/dotenv": {
"version": "17.4.1",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.1.tgz",
"integrity": "sha512-k8DaKGP6r1G30Lx8V4+pCsLzKr8vLmV2paqEj1Y55GdAgJuIqpRp5FfajGF8KtwMxCz9qJc6wUIJnm053d/WCw==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/dotenv-expand": {
"version": "12.0.3",
"resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-12.0.3.tgz",
"integrity": "sha512-uc47g4b+4k/M/SeaW1y4OApx+mtLWl92l5LMPP0GNXctZqELk+YGgOPIIC5elYmUH4OuoK3JLhuRUYegeySiFA==",
"license": "BSD-2-Clause",
"dependencies": {
"dotenv": "^16.4.5"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/dotenv-expand/node_modules/dotenv": {
"version": "16.6.1",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz",
"integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/dunder-proto": { "node_modules/dunder-proto": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -7375,6 +7320,7 @@
"version": "4.18.1", "version": "4.18.1",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz",
"integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==",
"dev": true,
"license": "MIT" "license": "MIT"
}, },
"node_modules/lodash.memoize": { "node_modules/lodash.memoize": {

View File

@@ -6,15 +6,14 @@
"license": "PERSONAL", "license": "PERSONAL",
"scripts": { "scripts": {
"build": "nest build", "build": "nest build",
"start": "nest start", "start": "node --env-file=.env ./node_modules/@nestjs/cli/bin/nest.js start",
"start:dev": "nest start --watch", "start:dev": "node --env-file=.env ./node_modules/@nestjs/cli/bin/nest.js start --watch",
"start:prod": "node dist/main.js", "start:prod": "node dist/main.js",
"lint": "eslint \"src/**/*.ts\"", "lint": "eslint \"src/**/*.ts\"",
"test": "jest --runInBand" "test": "jest --runInBand"
}, },
"dependencies": { "dependencies": {
"@nestjs/common": "^11.0.1", "@nestjs/common": "^11.0.1",
"@nestjs/config": "^4.0.2",
"@nestjs/core": "^11.0.1", "@nestjs/core": "^11.0.1",
"@nestjs/platform-express": "^11.0.1", "@nestjs/platform-express": "^11.0.1",
"class-transformer": "^0.5.1", "class-transformer": "^0.5.1",
@@ -25,22 +24,22 @@
"rxjs": "^7.8.1" "rxjs": "^7.8.1"
}, },
"devDependencies": { "devDependencies": {
"@eslint/js": "^9.18.0",
"@nestjs/cli": "^11.0.0", "@nestjs/cli": "^11.0.0",
"@types/compression": "^1.8.1", "@types/compression": "^1.8.1",
"@types/express": "^5.0.0", "@types/express": "^5.0.0",
"@types/jest": "^30.0.0", "@types/jest": "^30.0.0",
"@types/node": "^22.10.7", "@types/node": "^22.10.7",
"@types/pg": "^8.11.10", "@types/pg": "^8.11.10",
"eslint": "^9.18.0",
"globals": "^16.0.0",
"jest": "^30.0.0", "jest": "^30.0.0",
"ts-jest": "^29.2.5", "ts-jest": "^29.2.5",
"ts-loader": "^9.5.2", "ts-loader": "^9.5.2",
"ts-node": "^10.9.2", "ts-node": "^10.9.2",
"tsconfig-paths": "^4.2.0", "tsconfig-paths": "^4.2.0",
"typescript": "^5.7.3", "typescript": "^5.7.3",
"typescript-eslint": "^8.20.0", "typescript-eslint": "^8.20.0"
"@eslint/js": "^9.18.0",
"eslint": "^9.18.0",
"globals": "^16.0.0"
}, },
"jest": { "jest": {
"moduleFileExtensions": [ "moduleFileExtensions": [

View File

@@ -1,5 +1,4 @@
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
import { CurrentModule } from './current/current.module'; import { CurrentModule } from './current/current.module';
import { DbModule } from './db/db.module'; import { DbModule } from './db/db.module';
import { EdgeModule } from './edge/edge.module'; import { EdgeModule } from './edge/edge.module';
@@ -10,7 +9,6 @@ import { TagModule } from './tag/tag.module';
@Module({ @Module({
imports: [ imports: [
ConfigModule.forRoot({ isGlobal: true }),
DbModule, DbModule,
IngestModule, IngestModule,
EdgeModule, EdgeModule,

View File

@@ -1,25 +0,0 @@
import { ConfigService } from '@nestjs/config';
/** Читает числовой параметр конфига и использует резервное значение, если он пустой или неверный. */
export function getNumberConfig(
config: ConfigService,
key: string,
fallback: number,
): number {
const value = config.get<string>(key);
if (value === undefined || value === '') {
return fallback;
}
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : fallback;
}
/** Читает числовой параметр конфига и округляет вниз для лимитов и размеров пулов. */
export function getIntegerConfig(
config: ConfigService,
key: string,
fallback: number,
): number {
return Math.floor(getNumberConfig(config, key, fallback));
}

View File

@@ -1,14 +1,11 @@
import { CanActivate, ExecutionContext, Injectable, UnauthorizedException } from '@nestjs/common'; import { CanActivate, ExecutionContext, Injectable, UnauthorizedException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import type { Request } from 'express'; import type { Request } from 'express';
@Injectable() @Injectable()
export class IngestApiKeyGuard implements CanActivate { export class IngestApiKeyGuard implements CanActivate {
constructor(private readonly config: ConfigService) {}
/** По умолчанию открывает ingest, но проверяет x-api-key, если задан INGEST_API_KEY. */ /** По умолчанию открывает ingest, но проверяет x-api-key, если задан INGEST_API_KEY. */
canActivate(context: ExecutionContext): boolean { canActivate(context: ExecutionContext): boolean {
const expectedKey = this.config.get<string>('INGEST_API_KEY'); const expectedKey = process.env.INGEST_API_KEY;
if (!expectedKey) { if (!expectedKey) {
return true; return true;
} }

View File

@@ -1,22 +0,0 @@
import { BadRequestException } from '@nestjs/common';
// Единая нормализация обязательных строк: trim + понятная 400-ошибка для API.
/** Обрезает обязательное текстовое поле и возвращает 400, если оно стало пустым. */
export function normalizeRequiredText(value: string, fieldName: string): string {
const normalized = value.trim();
if (!normalized) {
throw new BadRequestException(`${fieldName} is required.`);
}
return normalized;
}
/** Обрезает необязательный query-текст и превращает пустые строки в undefined. */
export function normalizeOptionalText(value: unknown): string | undefined {
if (value === undefined || value === null) {
return undefined;
}
const normalized = String(value).trim();
return normalized || undefined;
}

View File

@@ -1,14 +0,0 @@
// Поддерживает оба формата: ?tags=a,b и ?tags=a&tags=b; дубликаты убираются.
/** Приводит повторяющиеся или разделенные запятыми query-параметры к чистому массиву строк. */
export function parseCommaSeparatedList(value: unknown): string[] | undefined {
if (value == null || value === '') {
return undefined;
}
const rawItems = Array.isArray(value)
? value.flatMap((item) => String(item).split(','))
: String(value).split(',');
const items = rawItems.map((item) => item.trim()).filter(Boolean);
return items.length ? Array.from(new Set(items)) : undefined;
}

View File

@@ -1,7 +1,5 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { distinctUntilChanged, from, map, Observable, switchMap, timer } from 'rxjs'; import { distinctUntilChanged, from, map, Observable, switchMap, timer } from 'rxjs';
import { getIntegerConfig } from '../common/config-number';
import { CurrentResponseDto } from './dto/current-response.dto'; import { CurrentResponseDto } from './dto/current-response.dto';
import { GetCurrentDto } from './dto/get-current.dto'; import { GetCurrentDto } from './dto/get-current.dto';
import { CurrentService } from './current.service'; import { CurrentService } from './current.service';
@@ -15,21 +13,13 @@ type CurrentSnapshot = {
response: CurrentResponseDto; response: CurrentResponseDto;
}; };
const DEFAULT_CURRENT_EVENTS_POLL_MS = 1_000;
@Injectable() @Injectable()
export class CurrentEventsService { export class CurrentEventsService {
constructor( constructor(private readonly current: CurrentService) {}
private readonly current: CurrentService,
private readonly config: ConfigService,
) {}
/** Отправляет SSE-события current только при изменении снимка данных по edge/tags. */ /** Отправляет SSE-события current только при изменении снимка данных по edge/tags. */
stream(query: GetCurrentDto): Observable<CurrentSseMessage> { stream(query: GetCurrentDto): Observable<CurrentSseMessage> {
const pollMs = Math.max( const pollMs = Number(process.env.CURRENT_EVENTS_POLL_MS);
getIntegerConfig(this.config, 'CURRENT_EVENTS_POLL_MS', DEFAULT_CURRENT_EVENTS_POLL_MS),
250,
);
return timer(0, pollMs).pipe( return timer(0, pollMs).pipe(
switchMap(() => from(this.createSnapshot(query))), switchMap(() => from(this.createSnapshot(query))),

View File

@@ -1,5 +1,4 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { normalizeRequiredText } from '../common/normalize-text';
import { CurrentResponseDto } from './dto/current-response.dto'; import { CurrentResponseDto } from './dto/current-response.dto';
import { GetCurrentDto } from './dto/get-current.dto'; import { GetCurrentDto } from './dto/get-current.dto';
import { createCurrentResponse } from './current.mapper'; import { createCurrentResponse } from './current.mapper';
@@ -11,7 +10,7 @@ export class CurrentService {
/** Нормализует фильтры и возвращает текущие значения по одному edge. */ /** Нормализует фильтры и возвращает текущие значения по одному edge. */
async findByEdge(query: GetCurrentDto): Promise<CurrentResponseDto> { async findByEdge(query: GetCurrentDto): Promise<CurrentResponseDto> {
const edge = normalizeRequiredText(query.edge, 'edge'); const edge = query.edge;
const tags = query.tags?.length ? query.tags : null; const tags = query.tags?.length ? query.tags : null;
const rows = await this.repository.findByEdge(edge, tags); const rows = await this.repository.findByEdge(edge, tags);
return createCurrentResponse(edge, rows); return createCurrentResponse(edge, rows);

View File

@@ -1,6 +1,4 @@
import { Transform } from 'class-transformer';
import { IsArray, IsNotEmpty, IsOptional, IsString } from 'class-validator'; import { IsArray, IsNotEmpty, IsOptional, IsString } from 'class-validator';
import { parseCommaSeparatedList } from '../../common/query-list';
export class GetCurrentDto { export class GetCurrentDto {
@IsNotEmpty() @IsNotEmpty()
@@ -8,7 +6,6 @@ export class GetCurrentDto {
edge!: string; edge!: string;
@IsOptional() @IsOptional()
@Transform(({ value }) => parseCommaSeparatedList(value))
@IsArray() @IsArray()
@IsString({ each: true }) @IsString({ each: true })
tags?: string[]; tags?: string[];

View File

@@ -1,7 +1,5 @@
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Pool, QueryResult, QueryResultRow } from 'pg'; import { Pool, QueryResult, QueryResultRow } from 'pg';
import { getIntegerConfig } from '../common/config-number';
type DatabaseHealth = { type DatabaseHealth = {
now: Date; now: Date;
@@ -13,18 +11,11 @@ type DatabaseHealth = {
export class DbService implements OnModuleInit, OnModuleDestroy { export class DbService implements OnModuleInit, OnModuleDestroy {
private pool!: Pool; private pool!: Pool;
constructor(private readonly config: ConfigService) {}
/** Создает пул PostgreSQL и проверяет БД до начала обработки запросов. */ /** Создает пул PostgreSQL и проверяет БД до начала обработки запросов. */
async onModuleInit(): Promise<void> { async onModuleInit(): Promise<void> {
const connectionString = this.config.get<string>('DATABASE_URL');
if (!connectionString) {
throw new Error('DATABASE_URL is required.');
}
this.pool = new Pool({ this.pool = new Pool({
connectionString, connectionString: process.env.DATABASE_URL,
max: getIntegerConfig(this.config, 'PG_POOL_MAX', 20), max: Number(process.env.PG_POOL_MAX),
idleTimeoutMillis: 30_000, idleTimeoutMillis: 30_000,
connectionTimeoutMillis: 5_000, connectionTimeoutMillis: 5_000,
statement_timeout: 60_000, statement_timeout: 60_000,

View File

@@ -4,9 +4,6 @@ export type EdgeItemDto = {
parentId: string | null; parentId: string | null;
tagIds: string[]; tagIds: string[];
tagCount: number; tagCount: number;
currentTagCount: number;
liveTagCount: number;
lastDataAt: Date | null;
}; };
export type EdgeResponseDto = { export type EdgeResponseDto = {

View File

@@ -1,15 +1,11 @@
import { Transform } from 'class-transformer';
import { IsOptional, IsString } from 'class-validator'; import { IsOptional, IsString } from 'class-validator';
import { normalizeOptionalText } from '../../common/normalize-text';
export class GetEdgesDto { export class GetEdgesDto {
@IsOptional() @IsOptional()
@Transform(({ value }) => normalizeOptionalText(value))
@IsString() @IsString()
parentId?: string; parentId?: string;
@IsOptional() @IsOptional()
@Transform(({ value }) => normalizeOptionalText(value))
@IsString() @IsString()
search?: string; search?: string;
} }

View File

@@ -10,9 +10,6 @@ export function createEdgeResponse(rows: EdgeRow[]): EdgeResponseDto {
parentId: row.parent_id, parentId: row.parent_id,
tagIds: row.tag_ids ?? [], tagIds: row.tag_ids ?? [],
tagCount: row.tag_count, tagCount: row.tag_count,
currentTagCount: row.current_tag_count,
liveTagCount: row.live_tag_count,
lastDataAt: row.last_data_at,
})), })),
}; };
} }

View File

@@ -6,7 +6,7 @@ import { EdgeRow } from './edge.types';
export class EdgeRepository { export class EdgeRepository {
constructor(private readonly db: DbService) {} constructor(private readonly db: DbService) {}
/** Читает каталог edge с легкими счетчиками свежести из текущих значений. */ /** Читает каталог edge без расчетов по текущим значениям. */
async findAll(parentId: string | null, search: string | null): Promise<EdgeRow[]> { async findAll(parentId: string | null, search: string | null): Promise<EdgeRow[]> {
const result = await this.db.query<EdgeRow>( const result = await this.db.query<EdgeRow>(
` `
@@ -15,23 +15,14 @@ export class EdgeRepository {
e.name, e.name,
e.parent_id, e.parent_id,
e.tag_ids, e.tag_ids,
cardinality(e.tag_ids)::integer AS tag_count, cardinality(e.tag_ids)::integer AS tag_count
count(DISTINCT c.tag)::integer AS current_tag_count,
count(DISTINCT c.tag) FILTER (
WHERE c."updatedAt" >= now() - INTERVAL '30 seconds'
)::integer AS live_tag_count,
max(c."updatedAt") AS last_data_at
FROM edge AS e FROM edge AS e
LEFT JOIN current AS c
ON c.edge = e.id
AND (cardinality(e.tag_ids) = 0 OR c.tag = ANY(e.tag_ids))
WHERE ($1::varchar IS NULL OR e.parent_id = $1::varchar) WHERE ($1::varchar IS NULL OR e.parent_id = $1::varchar)
AND ( AND (
$2::text IS NULL $2::text IS NULL
OR e.id ILIKE $2::text OR e.id ILIKE $2::text
OR e.name ILIKE $2::text OR e.name ILIKE $2::text
) )
GROUP BY e.id, e.name, e.parent_id, e.tag_ids
ORDER BY e.name ASC, e.id ASC ORDER BY e.name ASC, e.id ASC
`, `,
[parentId, search], [parentId, search],

View File

@@ -4,7 +4,4 @@ export type EdgeRow = {
parent_id: string | null; parent_id: string | null;
tag_ids: string[]; tag_ids: string[];
tag_count: number; tag_count: number;
current_tag_count: number;
live_tag_count: number;
last_data_at: Date | null;
}; };

View File

@@ -1,41 +1,23 @@
import { Transform, Type } from 'class-transformer'; import { IsDateString, IsNotEmpty, IsString } from 'class-validator';
import {
ArrayMaxSize,
IsArray,
IsDateString,
IsNotEmpty,
IsNumber,
IsOptional,
IsString,
Max,
Min,
} from 'class-validator';
import { parseCommaSeparatedList } from '../../common/query-list';
export class GetHistoryDto { export class GetHistoryDto {
@IsNotEmpty() @IsNotEmpty()
@IsString() @IsString()
edge!: string; edge!: string;
@IsOptional() @IsNotEmpty()
@Transform(({ value }) => parseCommaSeparatedList(value)) @IsString()
@IsArray() tag!: string;
@ArrayMaxSize(500)
@IsString({ each: true })
tags?: string[];
@IsOptional() @IsNotEmpty()
@IsDateString() @IsDateString()
from?: string; from!: string;
@IsOptional() @IsNotEmpty()
@IsDateString() @IsDateString()
to?: string; to!: string;
@IsOptional() @IsNotEmpty()
@Type(() => Number) @IsString()
@IsNumber() granulate!: string;
@Min(100)
@Max(5000)
targetPoints?: number;
} }

View File

@@ -1,6 +1,5 @@
export type HistoryPointDto = { export type HistoryPointDto = {
t: number; t: number;
v: number;
min: number; min: number;
avg: number; avg: number;
max: number; max: number;
@@ -13,14 +12,11 @@ export type HistorySeriesDto = {
points: HistoryPointDto[]; points: HistoryPointDto[];
}; };
export type HistoryResponseSource = 'empty' | 'latest' | 'raw' | '1m' | '5m' | '1h' | '1d';
export type HistoryResponseDto = { export type HistoryResponseDto = {
edge: string; edge: string;
source: HistoryResponseSource; tag: string;
targetPoints: number; from: Date;
to: Date;
granulate: string;
series: HistorySeriesDto[]; series: HistorySeriesDto[];
from?: Date;
to?: Date;
resolutionSeconds?: number | null;
}; };

View File

@@ -1,30 +0,0 @@
import { HistorySource } from './history.types';
export const RAW_HISTORY_SOURCE: HistorySource = {
name: 'raw',
table: 'history',
timeColumn: 'timestamp',
intervalSeconds: 0,
};
export const AGGREGATE_HISTORY_SOURCES: HistorySource[] = [
{ name: '1m', table: 'history_1m', timeColumn: 'bucket', intervalSeconds: 60 },
{ name: '5m', table: 'history_5m', timeColumn: 'bucket', intervalSeconds: 300 },
{ name: '1h', table: 'history_1h', timeColumn: 'bucket', intervalSeconds: 3600 },
{ name: '1d', table: 'history_1d', timeColumn: 'bucket', intervalSeconds: 86400 },
];
/** Подбирает самый детальный слой, который не превышает целевой бюджет точек. */
export function chooseHistorySource(from: Date, to: Date, targetPoints: number): HistorySource {
const durationSeconds = Math.max((to.getTime() - from.getTime()) / 1000, 1);
const desiredSeconds = durationSeconds / Math.max(targetPoints, 1);
if (desiredSeconds < 60) {
return RAW_HISTORY_SOURCE;
}
return (
AGGREGATE_HISTORY_SOURCES.find((source) => source.intervalSeconds >= desiredSeconds) ??
AGGREGATE_HISTORY_SOURCES[AGGREGATE_HISTORY_SOURCES.length - 1]
);
}

View File

@@ -1,55 +1,41 @@
import { import { HistoryResponseDto, HistorySeriesDto } from './dto/history-response.dto';
HistoryResponseDto,
HistoryResponseSource,
HistorySeriesDto,
} from './dto/history-response.dto';
import { HistoryRow } from './history.types'; import { HistoryRow } from './history.types';
type HistoryResponseOptions = { type HistoryResponseOptions = {
edge: string; edge: string;
source: HistoryResponseSource; tag: string;
targetPoints: number; from: Date;
rows?: HistoryRow[]; to: Date;
from?: Date; granulate: string;
to?: Date; rows: HistoryRow[];
resolutionSeconds?: number | null;
}; };
/** Превращает плоские строки истории в стабильный API-формат для графиков. */ /** Переводит строки агрегированной истории в формат графика min/avg/max. */
export function createHistoryResponse(options: HistoryResponseOptions): HistoryResponseDto { export function createHistoryResponse(options: HistoryResponseOptions): HistoryResponseDto {
return { return {
edge: options.edge, edge: options.edge,
source: options.source, tag: options.tag,
targetPoints: options.targetPoints,
series: mapHistoryRowsToSeries(options.rows ?? []),
from: options.from, from: options.from,
to: options.to, to: options.to,
resolutionSeconds: options.resolutionSeconds, granulate: options.granulate,
series: options.rows.length
? [
{
edge: options.edge,
tag: options.tag,
points: mapHistoryRowsToPoints(options.rows),
},
]
: [],
}; };
} }
/** Группирует строки БД по edge/tag и переводит Date в миллисекунды Unix. */ function mapHistoryRowsToPoints(rows: HistoryRow[]): HistorySeriesDto['points'] {
function mapHistoryRowsToSeries(rows: HistoryRow[]): HistorySeriesDto[] { return rows.map((row) => ({
const seriesMap = new Map<string, HistorySeriesDto>(); t: row.time.getTime(),
min: row.min_value,
for (const row of rows) { avg: row.avg_value,
const key = `${row.edge}\u0000${row.tag}`; max: row.max_value,
const series = seriesMap.get(key) ?? { count: row.point_count,
edge: row.edge, }));
tag: row.tag,
points: [],
};
series.points.push({
t: row.time.getTime(),
v: row.value,
min: row.min_value,
avg: row.avg_value,
max: row.max_value,
count: row.point_count,
});
seriesMap.set(key, series);
}
return Array.from(seriesMap.values());
} }

View File

@@ -1,160 +1,45 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { DbService } from '../db/db.service'; import { DbService } from '../db/db.service';
import { HistoryRow, HistorySource } from './history.types'; import { HistoryRow } from './history.types';
@Injectable() @Injectable()
export class HistoryRepository { export class HistoryRepository {
constructor(private readonly db: DbService) {} constructor(private readonly db: DbService) {}
/** Загружает последние N сырых точек истории для каждого запрошенного тега. */ /** Агрегирует сырую историю по одному тегу через TimescaleDB time_bucket. */
async findLatest(edge: string, tags: string[], targetPoints: number): Promise<HistoryRow[]> { async findBucketedRange(
const result = await this.db.query<HistoryRow>(
`
SELECT
latest.edge,
latest.tag,
latest.timestamp AS time,
latest.value,
latest.value AS min_value,
latest.value AS avg_value,
latest.value AS max_value,
1::integer AS point_count
FROM unnest($2::varchar[]) AS requested(tag)
CROSS JOIN LATERAL (
SELECT edge, tag, timestamp, value
FROM history
WHERE edge = $1
AND tag = requested.tag
ORDER BY timestamp DESC
LIMIT $3
) AS latest
ORDER BY latest.tag ASC, latest.timestamp ASC
`,
[edge, tags, targetPoints],
);
return result.rows;
}
/** Загружает сырую историю за диапазон и агрегирует ее в бакеты под бюджет точек. */
async findRawRange(
edge: string, edge: string,
tags: string[], tag: string,
from: Date, from: Date,
to: Date, to: Date,
targetPoints: number, granulate: string,
): Promise<HistoryRow[]> { ): Promise<HistoryRow[]> {
const bucketMillis = this.getBucketMillis(from, to, targetPoints);
const fromMillis = from.getTime();
const result = await this.db.query<HistoryRow>( const result = await this.db.query<HistoryRow>(
` `
WITH filtered AS (
SELECT
edge,
tag,
floor((extract(epoch FROM timestamp) * 1000 - $5) / $6)::bigint AS bucket_id,
value
FROM history
WHERE edge = $1
AND tag = ANY($2::varchar[])
AND timestamp >= $3
AND timestamp <= $4
)
SELECT SELECT
edge, $1::varchar AS edge,
tag, $2::varchar AS tag,
to_timestamp(($5 + bucket_id * $6) / 1000.0) AS time, bucket AS time,
avg(value)::double precision AS value,
min(value)::double precision AS min_value, min(value)::double precision AS min_value,
avg(value)::double precision AS avg_value, avg(value)::double precision AS avg_value,
max(value)::double precision AS max_value, max(value)::double precision AS max_value,
count(*)::integer AS point_count count(*)::integer AS point_count
FROM filtered
GROUP BY edge, tag, bucket_id
ORDER BY tag ASC, time ASC
`,
[edge, tags, from, to, fromMillis, bucketMillis],
);
return result.rows;
}
/** Читает заранее подготовленный aggregate-view с min/avg/max/count по бакетам. */
async findAggregateRange(
edge: string,
tags: string[],
from: Date,
to: Date,
source: HistorySource,
): Promise<HistoryRow[]> {
const result = await this.db.query<HistoryRow>(
`
SELECT
edge,
tag,
${source.timeColumn} AS time,
avg_value AS value,
min_value,
avg_value,
max_value,
point_count
FROM ${source.table}
WHERE edge = $1
AND tag = ANY($2::varchar[])
AND ${source.timeColumn} >= $3
AND ${source.timeColumn} <= $4
ORDER BY tag ASC, ${source.timeColumn} ASC
`,
[edge, tags, from, to],
);
return result.rows;
}
/** Рассчитывает ширину временного бакета так, чтобы на тег было не больше targetPoints точек. */
private getBucketMillis(from: Date, to: Date, targetPoints: number): number {
const durationMillis = Math.max(to.getTime() - from.getTime(), 1);
return Math.max(Math.ceil(durationMillis / Math.max(targetPoints, 1)), 1);
}
/** Берет теги из запроса или ищет связи edge, а затем резервно смотрит саму историю. */
async resolveTags(edge: string, requestedTags?: string[]): Promise<string[]> {
if (requestedTags?.length) {
return requestedTags;
}
const linked = await this.db.query<{ tag: string }>(
`
SELECT DISTINCT tag
FROM ( FROM (
SELECT unnest(e.tag_ids)::text AS tag SELECT
FROM edge AS e time_bucket($5::interval, timestamp) AS bucket,
WHERE e.id = $1 value
UNION FROM history
SELECT t.id AS tag WHERE edge = $1
FROM tag AS t AND tag = $2
WHERE $1 = ANY(t.edge_ids) AND timestamp >= $3
) AS linked_tags AND timestamp < $4
WHERE tag IS NOT NULL ) AS bucketed
ORDER BY tag ASC GROUP BY bucket
ORDER BY bucket ASC
`, `,
[edge], [edge, tag, from, to, granulate],
); );
if (linked.rows.length) { return result.rows;
return linked.rows.map((row) => row.tag);
}
const distinct = await this.db.query<{ tag: string }>(
`
SELECT DISTINCT tag
FROM history
WHERE edge = $1
ORDER BY tag ASC
`,
[edge],
);
return distinct.rows.map((row) => row.tag);
} }
} }

View File

@@ -1,66 +1,17 @@
import { BadRequestException, Injectable } from '@nestjs/common'; import { BadRequestException, Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { getIntegerConfig } from '../common/config-number';
import { normalizeRequiredText } from '../common/normalize-text';
import { GetHistoryDto } from './dto/get-history.dto'; import { GetHistoryDto } from './dto/get-history.dto';
import { HistoryResponseDto } from './dto/history-response.dto'; import { HistoryResponseDto } from './dto/history-response.dto';
import { chooseHistorySource } from './history-source';
import { createHistoryResponse } from './history.mapper'; import { createHistoryResponse } from './history.mapper';
import { HistoryRepository } from './history.repository'; import { HistoryRepository } from './history.repository';
@Injectable() @Injectable()
export class HistoryService { export class HistoryService {
constructor( constructor(private readonly repository: HistoryRepository) {}
private readonly repository: HistoryRepository,
private readonly config: ConfigService,
) {}
/** Определяет теги, выбирает слой истории и собирает API-ответ для графика. */ /** Получает агрегированную историю одного тега за диапазон с заданным шагом bucket. */
async findSeries(query: GetHistoryDto): Promise<HistoryResponseDto> { async findSeries(query: GetHistoryDto): Promise<HistoryResponseDto> {
const edge = normalizeRequiredText(query.edge, 'edge'); const from = new Date(query.from);
const targetPoints = this.getTargetPoints(query.targetPoints); const to = new Date(query.to);
const tags = await this.repository.resolveTags(edge, query.tags);
if (!tags.length) {
return createHistoryResponse({
edge,
source: 'empty',
targetPoints,
});
}
if (!query.from && !query.to) {
const rows = await this.repository.findLatest(edge, tags, targetPoints);
return createHistoryResponse({
edge,
source: 'latest',
targetPoints,
rows,
});
}
const { from, to } = this.parseRange(query);
const source = chooseHistorySource(from, to, targetPoints);
const rows =
source.name === 'raw'
? await this.repository.findRawRange(edge, tags, from, to, targetPoints)
: await this.repository.findAggregateRange(edge, tags, from, to, source);
return createHistoryResponse({
edge,
from,
to,
source: source.name,
resolutionSeconds: source.intervalSeconds || null,
targetPoints,
rows,
});
}
/** Разбирает опциональный диапазон времени и отклоняет неверные или обратные границы. */
private parseRange(query: GetHistoryDto): { from: Date; to: Date } {
const from = query.from ? new Date(query.from) : new Date(0);
const to = query.to ? new Date(query.to) : new Date();
if (Number.isNaN(from.getTime()) || Number.isNaN(to.getTime())) { if (Number.isNaN(from.getTime()) || Number.isNaN(to.getTime())) {
throw new BadRequestException('Invalid from/to date.'); throw new BadRequestException('Invalid from/to date.');
@@ -70,14 +21,21 @@ export class HistoryService {
throw new BadRequestException('from cannot be greater than to.'); throw new BadRequestException('from cannot be greater than to.');
} }
return { from, to }; const rows = await this.repository.findBucketedRange(
} query.edge,
query.tag,
from,
to,
query.granulate,
);
/** Применяет лимиты точек по умолчанию и максимуму, чтобы ответ истории был удобным для графиков. */ return createHistoryResponse({
private getTargetPoints(requested?: number): number { edge: query.edge,
const defaultValue = getIntegerConfig(this.config, 'HISTORY_TARGET_POINTS', 2000); tag: query.tag,
const maxValue = getIntegerConfig(this.config, 'HISTORY_MAX_TARGET_POINTS', 5000); from,
const value = requested ?? defaultValue; to,
return Math.min(Math.max(Math.floor(value), 100), maxValue); granulate: query.granulate,
rows,
});
} }
} }

View File

@@ -1,17 +1,7 @@
export type HistorySourceName = 'raw' | '1m' | '5m' | '1h' | '1d';
export type HistorySource = {
name: HistorySourceName;
table: string;
timeColumn: string;
intervalSeconds: number;
};
export type HistoryRow = { export type HistoryRow = {
edge: string; edge: string;
tag: string; tag: string;
time: Date; time: Date;
value: number;
min_value: number; min_value: number;
avg_value: number; avg_value: number;
max_value: number; max_value: number;

View File

@@ -1,12 +1,7 @@
import { BadRequestException, Injectable } from '@nestjs/common'; import { BadRequestException, Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { getIntegerConfig } from '../common/config-number';
import { normalizeRequiredText } from '../common/normalize-text';
import { DbService } from '../db/db.service'; import { DbService } from '../db/db.service';
import { IngestPointDto } from './dto/ingest-point.dto'; import { IngestPointDto } from './dto/ingest-point.dto';
const DEFAULT_MAX_BATCH_SIZE = 10_000;
type NormalizedPoint = { type NormalizedPoint = {
time: Date; time: Date;
edge: string; edge: string;
@@ -20,10 +15,7 @@ type IngestResult = {
@Injectable() @Injectable()
export class IngestService { export class IngestService {
constructor( constructor(private readonly db: DbService) {}
private readonly db: DbService,
private readonly config: ConfigService,
) {}
/** Проверяет батч, пишет сырую историю и обновляет последний снимок current для UI/SSE. */ /** Проверяет батч, пишет сырую историю и обновляет последний снимок current для UI/SSE. */
async ingestPoints(points: IngestPointDto[]): Promise<IngestResult> { async ingestPoints(points: IngestPointDto[]): Promise<IngestResult> {
@@ -89,10 +81,7 @@ export class IngestService {
/** Защищает API от слишком больших ingest-запросов, занимающих соединение с БД надолго. */ /** Защищает API от слишком больших ingest-запросов, занимающих соединение с БД надолго. */
private assertBatchSize(size: number): void { private assertBatchSize(size: number): void {
const maxBatchSize = Math.max( const maxBatchSize = Number(process.env.INGEST_MAX_BATCH_SIZE);
getIntegerConfig(this.config, 'INGEST_MAX_BATCH_SIZE', DEFAULT_MAX_BATCH_SIZE),
1,
);
if (size > maxBatchSize) { if (size > maxBatchSize) {
throw new BadRequestException(`Batch size must not exceed ${maxBatchSize} points.`); throw new BadRequestException(`Batch size must not exceed ${maxBatchSize} points.`);
@@ -101,17 +90,11 @@ export class IngestService {
/** Приводит входной DTO к нормализованному виду для вставки в history. */ /** Приводит входной DTO к нормализованному виду для вставки в history. */
private normalizePoint(point: IngestPointDto): NormalizedPoint { private normalizePoint(point: IngestPointDto): NormalizedPoint {
const edge = normalizeRequiredText(point.edge, 'edge');
const tag = normalizeRequiredText(point.tag, 'tag');
const time = this.parsePointTime(point); const time = this.parsePointTime(point);
if (!Number.isFinite(point.value)) {
throw new BadRequestException('value must be a finite number.');
}
return { return {
edge, edge: point.edge,
tag, tag: point.tag,
value: point.value, value: point.value,
time, time,
}; };

View File

@@ -8,12 +8,8 @@ async function bootstrap() {
const app = await NestFactory.create(AppModule); const app = await NestFactory.create(AppModule);
app.use(compression()); app.use(compression());
const corsAllowedOrigins = process.env.CORS_ALLOWED_ORIGINS
? process.env.CORS_ALLOWED_ORIGINS.split(',').map((origin) => origin.trim()).filter(Boolean)
: true;
app.enableCors({ app.enableCors({
origin: corsAllowedOrigins, origin: process.env.CORS_ALLOWED_ORIGINS,
credentials: true, credentials: true,
methods: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], methods: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'],
allowedHeaders: ['Content-Type', 'Authorization', 'x-api-key'], allowedHeaders: ['Content-Type', 'Authorization', 'x-api-key'],
@@ -27,7 +23,7 @@ async function bootstrap() {
}), }),
); );
await app.listen(process.env.PORT ?? 3101); await app.listen(process.env.PORT as string);
} }
void bootstrap(); void bootstrap();

View File

@@ -1,15 +1,11 @@
import { Transform } from 'class-transformer';
import { IsOptional, IsString } from 'class-validator'; import { IsOptional, IsString } from 'class-validator';
import { normalizeOptionalText } from '../../common/normalize-text';
export class GetTagsDto { export class GetTagsDto {
@IsOptional() @IsOptional()
@Transform(({ value }) => normalizeOptionalText(value))
@IsString() @IsString()
edge?: string; edge?: string;
@IsOptional() @IsOptional()
@Transform(({ value }) => normalizeOptionalText(value))
@IsString() @IsString()
search?: string; search?: string;
} }