From a12d490513f9899244c197c1e1222d836ac20ff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B2=D0=BE=D0=B2=20=D0=90=D1=80=D1=82?= =?UTF-8?q?=D0=B5=D0=BC?= Date: Fri, 19 Jun 2026 14:59:39 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A3=D0=B1=D1=80=D0=B0=D0=BD=20=D0=BB=D0=B8?= =?UTF-8?q?=D1=88=D0=BD=D0=B8=D0=B9=20=D0=BA=D0=BE=D0=B4.=20=D0=A1=D0=BA?= =?UTF-8?q?=D0=BE=D1=80=D1=80=D0=B5=D0=BA=D1=82=D0=B8=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B0=D0=BD=D1=8B=20docker=20=D0=B8=20package=20=D1=84=D0=B0?= =?UTF-8?q?=D0=B9=D0=BB=D1=8B.=20=D0=9F=D0=B5=D1=80=D0=B5=D1=80=D0=B0?= =?UTF-8?q?=D0=B1=D0=BE=D1=82=D0=B0=D0=BD=D0=BE=20=D0=B2=D0=B7=D0=B0=D0=B8?= =?UTF-8?q?=D0=BC=D0=BE=D0=B4=D0=B5=D0=B9=D1=81=D1=82=D0=B2=D0=B8=D0=B5=20?= =?UTF-8?q?=D1=81=20=D0=B1=D0=B4=20-=20=D1=83=D0=BF=D1=80=D0=BE=D1=89?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0=20=D0=BB=D0=BE=D0=B3=D0=B8=D0=BA=D0=B0.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 4 + .env.example | 4 - .env.local.example | 2 - Dockerfile | 18 ++- docker-compose.yml | 16 ++- package-lock.json | 56 +-------- package.json | 13 +- src/app.module.ts | 2 - src/common/config-number.ts | 25 ---- src/common/ingest-api-key.guard.ts | 5 +- src/common/normalize-text.ts | 22 ---- src/common/query-list.ts | 14 --- src/current/current-events.service.ts | 14 +-- src/current/current.service.ts | 3 +- src/current/dto/get-current.dto.ts | 3 - src/db/db.service.ts | 13 +- src/edge/dto/edge-response.dto.ts | 3 - src/edge/dto/get-edges.dto.ts | 4 - src/edge/edge.mapper.ts | 3 - src/edge/edge.repository.ts | 13 +- src/edge/edge.types.ts | 3 - src/history/dto/get-history.dto.ts | 40 ++---- src/history/dto/history-response.dto.ts | 12 +- src/history/history-source.ts | 30 ----- src/history/history.mapper.ts | 66 ++++------ src/history/history.repository.ts | 157 ++++-------------------- src/history/history.service.ts | 80 +++--------- src/history/history.types.ts | 10 -- src/ingest/ingest.service.ts | 25 +--- src/main.ts | 8 +- src/tag/dto/get-tags.dto.ts | 4 - 31 files changed, 120 insertions(+), 552 deletions(-) create mode 100644 .dockerignore delete mode 100644 src/common/config-number.ts delete mode 100644 src/common/normalize-text.ts delete mode 100644 src/common/query-list.ts delete mode 100644 src/history/history-source.ts diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..69e11c9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +node_modules +dist +.env +.env.* diff --git a/.env.example b/.env.example index 6ef862e..2f89709 100644 --- a/.env.example +++ b/.env.example @@ -13,7 +13,3 @@ INGEST_MAX_BATCH_SIZE=10000 # Poll interval for GET /current/events SSE snapshots. CURRENT_EVENTS_POLL_MS=1000 - -# Graph API defaults. -HISTORY_TARGET_POINTS=2000 -HISTORY_MAX_TARGET_POINTS=5000 diff --git a/.env.local.example b/.env.local.example index 1e57228..064a209 100644 --- a/.env.local.example +++ b/.env.local.example @@ -5,5 +5,3 @@ INGEST_API_KEY= PG_POOL_MAX=20 INGEST_MAX_BATCH_SIZE=10000 CURRENT_EVENTS_POLL_MS=1000 -HISTORY_TARGET_POINTS=2000 -HISTORY_MAX_TARGET_POINTS=5000 diff --git a/Dockerfile b/Dockerfile index 4934f58..189c16d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,14 @@ -FROM node:22-alpine AS deps +FROM node:22-alpine + WORKDIR /app + COPY package*.json ./ RUN npm ci -FROM node:22-alpine AS build -WORKDIR /app -COPY --from=deps /app/node_modules ./node_modules -COPY . . -RUN npm run build +COPY tsconfig*.json ./ +COPY src ./src + +RUN npm run build && npm prune --omit=dev -FROM node:22-alpine AS runtime -WORKDIR /app ENV NODE_ENV=production -COPY package*.json ./ -RUN npm ci --omit=dev -COPY --from=build /app/dist ./dist CMD ["node", "dist/main.js"] diff --git a/docker-compose.yml b/docker-compose.yml index 52c114b..e0aae92 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,17 +5,15 @@ services: restart: unless-stopped environment: NODE_ENV: production - PORT: ${PORT:-3101} + PORT: ${PORT} DATABASE_URL: ${DATABASE_URL} - CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:-} - INGEST_API_KEY: ${INGEST_API_KEY:-} - PG_POOL_MAX: ${PG_POOL_MAX:-20} - INGEST_MAX_BATCH_SIZE: ${INGEST_MAX_BATCH_SIZE:-10000} - CURRENT_EVENTS_POLL_MS: ${CURRENT_EVENTS_POLL_MS:-1000} - HISTORY_TARGET_POINTS: ${HISTORY_TARGET_POINTS:-2000} - HISTORY_MAX_TARGET_POINTS: ${HISTORY_MAX_TARGET_POINTS:-5000} + CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS} + INGEST_API_KEY: ${INGEST_API_KEY} + PG_POOL_MAX: ${PG_POOL_MAX} + INGEST_MAX_BATCH_SIZE: ${INGEST_MAX_BATCH_SIZE} + CURRENT_EVENTS_POLL_MS: ${CURRENT_EVENTS_POLL_MS} ports: - - "${PORT:-3101}:${PORT:-3101}" + - "${PORT}:${PORT}" networks: - proxy diff --git a/package-lock.json b/package-lock.json index 069c57a..58c3b1f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,6 @@ "license": "PERSONAL", "dependencies": { "@nestjs/common": "^11.0.1", - "@nestjs/config": "^4.0.2", "@nestjs/core": "^11.0.1", "@nestjs/platform-express": "^11.0.1", "class-transformer": "^0.5.1", @@ -2359,21 +2358,6 @@ } } }, - "node_modules/@nestjs/config": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/@nestjs/config/-/config-4.0.4.tgz", - "integrity": "sha512-CJPjNitr0bAufSEnRe2N+JbnVmMmDoo6hvKCPzXgZoGwJSmp/dZPk9f/RMbuD/+Q1ZJPjwsRpq0vxna++Knwow==", - "license": "MIT", - "dependencies": { - "dotenv": "17.4.1", - "dotenv-expand": "12.0.3", - "lodash": "4.18.1" - }, - "peerDependencies": { - "@nestjs/common": "^10.0.0 || ^11.0.0", - "rxjs": "^7.1.0" - } - }, "node_modules/@nestjs/core": { "version": "11.1.26", "resolved": "https://registry.npmjs.org/@nestjs/core/-/core-11.1.26.tgz", @@ -4938,45 +4922,6 @@ "node": ">=0.3.1" } }, - "node_modules/dotenv": { - "version": "17.4.1", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.1.tgz", - "integrity": "sha512-k8DaKGP6r1G30Lx8V4+pCsLzKr8vLmV2paqEj1Y55GdAgJuIqpRp5FfajGF8KtwMxCz9qJc6wUIJnm053d/WCw==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/dotenv-expand": { - "version": "12.0.3", - "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-12.0.3.tgz", - "integrity": "sha512-uc47g4b+4k/M/SeaW1y4OApx+mtLWl92l5LMPP0GNXctZqELk+YGgOPIIC5elYmUH4OuoK3JLhuRUYegeySiFA==", - "license": "BSD-2-Clause", - "dependencies": { - "dotenv": "^16.4.5" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/dotenv-expand/node_modules/dotenv": { - "version": "16.6.1", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", - "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -7375,6 +7320,7 @@ "version": "4.18.1", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "dev": true, "license": "MIT" }, "node_modules/lodash.memoize": { diff --git a/package.json b/package.json index 85c43ce..5da45f8 100644 --- a/package.json +++ b/package.json @@ -6,15 +6,14 @@ "license": "PERSONAL", "scripts": { "build": "nest build", - "start": "nest start", - "start:dev": "nest start --watch", + "start": "node --env-file=.env ./node_modules/@nestjs/cli/bin/nest.js start", + "start:dev": "node --env-file=.env ./node_modules/@nestjs/cli/bin/nest.js start --watch", "start:prod": "node dist/main.js", "lint": "eslint \"src/**/*.ts\"", "test": "jest --runInBand" }, "dependencies": { "@nestjs/common": "^11.0.1", - "@nestjs/config": "^4.0.2", "@nestjs/core": "^11.0.1", "@nestjs/platform-express": "^11.0.1", "class-transformer": "^0.5.1", @@ -25,22 +24,22 @@ "rxjs": "^7.8.1" }, "devDependencies": { + "@eslint/js": "^9.18.0", "@nestjs/cli": "^11.0.0", "@types/compression": "^1.8.1", "@types/express": "^5.0.0", "@types/jest": "^30.0.0", "@types/node": "^22.10.7", "@types/pg": "^8.11.10", + "eslint": "^9.18.0", + "globals": "^16.0.0", "jest": "^30.0.0", "ts-jest": "^29.2.5", "ts-loader": "^9.5.2", "ts-node": "^10.9.2", "tsconfig-paths": "^4.2.0", "typescript": "^5.7.3", - "typescript-eslint": "^8.20.0", - "@eslint/js": "^9.18.0", - "eslint": "^9.18.0", - "globals": "^16.0.0" + "typescript-eslint": "^8.20.0" }, "jest": { "moduleFileExtensions": [ diff --git a/src/app.module.ts b/src/app.module.ts index 91f17fc..8c1f1b8 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -1,5 +1,4 @@ import { Module } from '@nestjs/common'; -import { ConfigModule } from '@nestjs/config'; import { CurrentModule } from './current/current.module'; import { DbModule } from './db/db.module'; import { EdgeModule } from './edge/edge.module'; @@ -10,7 +9,6 @@ import { TagModule } from './tag/tag.module'; @Module({ imports: [ - ConfigModule.forRoot({ isGlobal: true }), DbModule, IngestModule, EdgeModule, diff --git a/src/common/config-number.ts b/src/common/config-number.ts deleted file mode 100644 index f3b264f..0000000 --- a/src/common/config-number.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { ConfigService } from '@nestjs/config'; - -/** Читает числовой параметр конфига и использует резервное значение, если он пустой или неверный. */ -export function getNumberConfig( - config: ConfigService, - key: string, - fallback: number, -): number { - const value = config.get(key); - if (value === undefined || value === '') { - return fallback; - } - - const parsed = Number(value); - return Number.isFinite(parsed) ? parsed : fallback; -} - -/** Читает числовой параметр конфига и округляет вниз для лимитов и размеров пулов. */ -export function getIntegerConfig( - config: ConfigService, - key: string, - fallback: number, -): number { - return Math.floor(getNumberConfig(config, key, fallback)); -} diff --git a/src/common/ingest-api-key.guard.ts b/src/common/ingest-api-key.guard.ts index 9f602af..5abebab 100644 --- a/src/common/ingest-api-key.guard.ts +++ b/src/common/ingest-api-key.guard.ts @@ -1,14 +1,11 @@ import { CanActivate, ExecutionContext, Injectable, UnauthorizedException } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import type { Request } from 'express'; @Injectable() export class IngestApiKeyGuard implements CanActivate { - constructor(private readonly config: ConfigService) {} - /** По умолчанию открывает ingest, но проверяет x-api-key, если задан INGEST_API_KEY. */ canActivate(context: ExecutionContext): boolean { - const expectedKey = this.config.get('INGEST_API_KEY'); + const expectedKey = process.env.INGEST_API_KEY; if (!expectedKey) { return true; } diff --git a/src/common/normalize-text.ts b/src/common/normalize-text.ts deleted file mode 100644 index c7955c5..0000000 --- a/src/common/normalize-text.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { BadRequestException } from '@nestjs/common'; - -// Единая нормализация обязательных строк: trim + понятная 400-ошибка для API. -/** Обрезает обязательное текстовое поле и возвращает 400, если оно стало пустым. */ -export function normalizeRequiredText(value: string, fieldName: string): string { - const normalized = value.trim(); - if (!normalized) { - throw new BadRequestException(`${fieldName} is required.`); - } - - return normalized; -} - -/** Обрезает необязательный query-текст и превращает пустые строки в undefined. */ -export function normalizeOptionalText(value: unknown): string | undefined { - if (value === undefined || value === null) { - return undefined; - } - - const normalized = String(value).trim(); - return normalized || undefined; -} diff --git a/src/common/query-list.ts b/src/common/query-list.ts deleted file mode 100644 index acee713..0000000 --- a/src/common/query-list.ts +++ /dev/null @@ -1,14 +0,0 @@ -// Поддерживает оба формата: ?tags=a,b и ?tags=a&tags=b; дубликаты убираются. -/** Приводит повторяющиеся или разделенные запятыми query-параметры к чистому массиву строк. */ -export function parseCommaSeparatedList(value: unknown): string[] | undefined { - if (value == null || value === '') { - return undefined; - } - - const rawItems = Array.isArray(value) - ? value.flatMap((item) => String(item).split(',')) - : String(value).split(','); - - const items = rawItems.map((item) => item.trim()).filter(Boolean); - return items.length ? Array.from(new Set(items)) : undefined; -} diff --git a/src/current/current-events.service.ts b/src/current/current-events.service.ts index e04e94b..c2ed6ef 100644 --- a/src/current/current-events.service.ts +++ b/src/current/current-events.service.ts @@ -1,7 +1,5 @@ import { Injectable } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import { distinctUntilChanged, from, map, Observable, switchMap, timer } from 'rxjs'; -import { getIntegerConfig } from '../common/config-number'; import { CurrentResponseDto } from './dto/current-response.dto'; import { GetCurrentDto } from './dto/get-current.dto'; import { CurrentService } from './current.service'; @@ -15,21 +13,13 @@ type CurrentSnapshot = { response: CurrentResponseDto; }; -const DEFAULT_CURRENT_EVENTS_POLL_MS = 1_000; - @Injectable() export class CurrentEventsService { - constructor( - private readonly current: CurrentService, - private readonly config: ConfigService, - ) {} + constructor(private readonly current: CurrentService) {} /** Отправляет SSE-события current только при изменении снимка данных по edge/tags. */ stream(query: GetCurrentDto): Observable { - const pollMs = Math.max( - getIntegerConfig(this.config, 'CURRENT_EVENTS_POLL_MS', DEFAULT_CURRENT_EVENTS_POLL_MS), - 250, - ); + const pollMs = Number(process.env.CURRENT_EVENTS_POLL_MS); return timer(0, pollMs).pipe( switchMap(() => from(this.createSnapshot(query))), diff --git a/src/current/current.service.ts b/src/current/current.service.ts index 6ceb7a5..5c7848a 100644 --- a/src/current/current.service.ts +++ b/src/current/current.service.ts @@ -1,5 +1,4 @@ import { Injectable } from '@nestjs/common'; -import { normalizeRequiredText } from '../common/normalize-text'; import { CurrentResponseDto } from './dto/current-response.dto'; import { GetCurrentDto } from './dto/get-current.dto'; import { createCurrentResponse } from './current.mapper'; @@ -11,7 +10,7 @@ export class CurrentService { /** Нормализует фильтры и возвращает текущие значения по одному edge. */ async findByEdge(query: GetCurrentDto): Promise { - const edge = normalizeRequiredText(query.edge, 'edge'); + const edge = query.edge; const tags = query.tags?.length ? query.tags : null; const rows = await this.repository.findByEdge(edge, tags); return createCurrentResponse(edge, rows); diff --git a/src/current/dto/get-current.dto.ts b/src/current/dto/get-current.dto.ts index d232da6..a6e0056 100644 --- a/src/current/dto/get-current.dto.ts +++ b/src/current/dto/get-current.dto.ts @@ -1,6 +1,4 @@ -import { Transform } from 'class-transformer'; import { IsArray, IsNotEmpty, IsOptional, IsString } from 'class-validator'; -import { parseCommaSeparatedList } from '../../common/query-list'; export class GetCurrentDto { @IsNotEmpty() @@ -8,7 +6,6 @@ export class GetCurrentDto { edge!: string; @IsOptional() - @Transform(({ value }) => parseCommaSeparatedList(value)) @IsArray() @IsString({ each: true }) tags?: string[]; diff --git a/src/db/db.service.ts b/src/db/db.service.ts index e38f9ab..6e2a59d 100644 --- a/src/db/db.service.ts +++ b/src/db/db.service.ts @@ -1,7 +1,5 @@ import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import { Pool, QueryResult, QueryResultRow } from 'pg'; -import { getIntegerConfig } from '../common/config-number'; type DatabaseHealth = { now: Date; @@ -13,18 +11,11 @@ type DatabaseHealth = { export class DbService implements OnModuleInit, OnModuleDestroy { private pool!: Pool; - constructor(private readonly config: ConfigService) {} - /** Создает пул PostgreSQL и проверяет БД до начала обработки запросов. */ async onModuleInit(): Promise { - const connectionString = this.config.get('DATABASE_URL'); - if (!connectionString) { - throw new Error('DATABASE_URL is required.'); - } - this.pool = new Pool({ - connectionString, - max: getIntegerConfig(this.config, 'PG_POOL_MAX', 20), + connectionString: process.env.DATABASE_URL, + max: Number(process.env.PG_POOL_MAX), idleTimeoutMillis: 30_000, connectionTimeoutMillis: 5_000, statement_timeout: 60_000, diff --git a/src/edge/dto/edge-response.dto.ts b/src/edge/dto/edge-response.dto.ts index a0274fe..76bdb4d 100644 --- a/src/edge/dto/edge-response.dto.ts +++ b/src/edge/dto/edge-response.dto.ts @@ -4,9 +4,6 @@ export type EdgeItemDto = { parentId: string | null; tagIds: string[]; tagCount: number; - currentTagCount: number; - liveTagCount: number; - lastDataAt: Date | null; }; export type EdgeResponseDto = { diff --git a/src/edge/dto/get-edges.dto.ts b/src/edge/dto/get-edges.dto.ts index 8004e4d..a5f25ef 100644 --- a/src/edge/dto/get-edges.dto.ts +++ b/src/edge/dto/get-edges.dto.ts @@ -1,15 +1,11 @@ -import { Transform } from 'class-transformer'; import { IsOptional, IsString } from 'class-validator'; -import { normalizeOptionalText } from '../../common/normalize-text'; export class GetEdgesDto { @IsOptional() - @Transform(({ value }) => normalizeOptionalText(value)) @IsString() parentId?: string; @IsOptional() - @Transform(({ value }) => normalizeOptionalText(value)) @IsString() search?: string; } diff --git a/src/edge/edge.mapper.ts b/src/edge/edge.mapper.ts index da0fa58..0ce411f 100644 --- a/src/edge/edge.mapper.ts +++ b/src/edge/edge.mapper.ts @@ -10,9 +10,6 @@ export function createEdgeResponse(rows: EdgeRow[]): EdgeResponseDto { parentId: row.parent_id, tagIds: row.tag_ids ?? [], tagCount: row.tag_count, - currentTagCount: row.current_tag_count, - liveTagCount: row.live_tag_count, - lastDataAt: row.last_data_at, })), }; } diff --git a/src/edge/edge.repository.ts b/src/edge/edge.repository.ts index fea71f3..077180d 100644 --- a/src/edge/edge.repository.ts +++ b/src/edge/edge.repository.ts @@ -6,7 +6,7 @@ import { EdgeRow } from './edge.types'; export class EdgeRepository { constructor(private readonly db: DbService) {} - /** Читает каталог edge с легкими счетчиками свежести из текущих значений. */ + /** Читает каталог edge без расчетов по текущим значениям. */ async findAll(parentId: string | null, search: string | null): Promise { const result = await this.db.query( ` @@ -15,23 +15,14 @@ export class EdgeRepository { e.name, e.parent_id, e.tag_ids, - cardinality(e.tag_ids)::integer AS tag_count, - count(DISTINCT c.tag)::integer AS current_tag_count, - count(DISTINCT c.tag) FILTER ( - WHERE c."updatedAt" >= now() - INTERVAL '30 seconds' - )::integer AS live_tag_count, - max(c."updatedAt") AS last_data_at + cardinality(e.tag_ids)::integer AS tag_count FROM edge AS e - LEFT JOIN current AS c - ON c.edge = e.id - AND (cardinality(e.tag_ids) = 0 OR c.tag = ANY(e.tag_ids)) WHERE ($1::varchar IS NULL OR e.parent_id = $1::varchar) AND ( $2::text IS NULL OR e.id ILIKE $2::text OR e.name ILIKE $2::text ) - GROUP BY e.id, e.name, e.parent_id, e.tag_ids ORDER BY e.name ASC, e.id ASC `, [parentId, search], diff --git a/src/edge/edge.types.ts b/src/edge/edge.types.ts index d9ce984..9d3d41d 100644 --- a/src/edge/edge.types.ts +++ b/src/edge/edge.types.ts @@ -4,7 +4,4 @@ export type EdgeRow = { parent_id: string | null; tag_ids: string[]; tag_count: number; - current_tag_count: number; - live_tag_count: number; - last_data_at: Date | null; }; diff --git a/src/history/dto/get-history.dto.ts b/src/history/dto/get-history.dto.ts index 65d845f..20bf371 100644 --- a/src/history/dto/get-history.dto.ts +++ b/src/history/dto/get-history.dto.ts @@ -1,41 +1,23 @@ -import { Transform, Type } from 'class-transformer'; -import { - ArrayMaxSize, - IsArray, - IsDateString, - IsNotEmpty, - IsNumber, - IsOptional, - IsString, - Max, - Min, -} from 'class-validator'; -import { parseCommaSeparatedList } from '../../common/query-list'; +import { IsDateString, IsNotEmpty, IsString } from 'class-validator'; export class GetHistoryDto { @IsNotEmpty() @IsString() edge!: string; - @IsOptional() - @Transform(({ value }) => parseCommaSeparatedList(value)) - @IsArray() - @ArrayMaxSize(500) - @IsString({ each: true }) - tags?: string[]; + @IsNotEmpty() + @IsString() + tag!: string; - @IsOptional() + @IsNotEmpty() @IsDateString() - from?: string; + from!: string; - @IsOptional() + @IsNotEmpty() @IsDateString() - to?: string; + to!: string; - @IsOptional() - @Type(() => Number) - @IsNumber() - @Min(100) - @Max(5000) - targetPoints?: number; + @IsNotEmpty() + @IsString() + granulate!: string; } diff --git a/src/history/dto/history-response.dto.ts b/src/history/dto/history-response.dto.ts index 1719f0a..a28cc2c 100644 --- a/src/history/dto/history-response.dto.ts +++ b/src/history/dto/history-response.dto.ts @@ -1,6 +1,5 @@ export type HistoryPointDto = { t: number; - v: number; min: number; avg: number; max: number; @@ -13,14 +12,11 @@ export type HistorySeriesDto = { points: HistoryPointDto[]; }; -export type HistoryResponseSource = 'empty' | 'latest' | 'raw' | '1m' | '5m' | '1h' | '1d'; - export type HistoryResponseDto = { edge: string; - source: HistoryResponseSource; - targetPoints: number; + tag: string; + from: Date; + to: Date; + granulate: string; series: HistorySeriesDto[]; - from?: Date; - to?: Date; - resolutionSeconds?: number | null; }; diff --git a/src/history/history-source.ts b/src/history/history-source.ts deleted file mode 100644 index cd0167d..0000000 --- a/src/history/history-source.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { HistorySource } from './history.types'; - -export const RAW_HISTORY_SOURCE: HistorySource = { - name: 'raw', - table: 'history', - timeColumn: 'timestamp', - intervalSeconds: 0, -}; - -export const AGGREGATE_HISTORY_SOURCES: HistorySource[] = [ - { name: '1m', table: 'history_1m', timeColumn: 'bucket', intervalSeconds: 60 }, - { name: '5m', table: 'history_5m', timeColumn: 'bucket', intervalSeconds: 300 }, - { name: '1h', table: 'history_1h', timeColumn: 'bucket', intervalSeconds: 3600 }, - { name: '1d', table: 'history_1d', timeColumn: 'bucket', intervalSeconds: 86400 }, -]; - -/** Подбирает самый детальный слой, который не превышает целевой бюджет точек. */ -export function chooseHistorySource(from: Date, to: Date, targetPoints: number): HistorySource { - const durationSeconds = Math.max((to.getTime() - from.getTime()) / 1000, 1); - const desiredSeconds = durationSeconds / Math.max(targetPoints, 1); - - if (desiredSeconds < 60) { - return RAW_HISTORY_SOURCE; - } - - return ( - AGGREGATE_HISTORY_SOURCES.find((source) => source.intervalSeconds >= desiredSeconds) ?? - AGGREGATE_HISTORY_SOURCES[AGGREGATE_HISTORY_SOURCES.length - 1] - ); -} diff --git a/src/history/history.mapper.ts b/src/history/history.mapper.ts index 64c49b5..c481fab 100644 --- a/src/history/history.mapper.ts +++ b/src/history/history.mapper.ts @@ -1,55 +1,41 @@ -import { - HistoryResponseDto, - HistoryResponseSource, - HistorySeriesDto, -} from './dto/history-response.dto'; +import { HistoryResponseDto, HistorySeriesDto } from './dto/history-response.dto'; import { HistoryRow } from './history.types'; type HistoryResponseOptions = { edge: string; - source: HistoryResponseSource; - targetPoints: number; - rows?: HistoryRow[]; - from?: Date; - to?: Date; - resolutionSeconds?: number | null; + tag: string; + from: Date; + to: Date; + granulate: string; + rows: HistoryRow[]; }; -/** Превращает плоские строки истории в стабильный API-формат для графиков. */ +/** Переводит строки агрегированной истории в формат графика min/avg/max. */ export function createHistoryResponse(options: HistoryResponseOptions): HistoryResponseDto { return { edge: options.edge, - source: options.source, - targetPoints: options.targetPoints, - series: mapHistoryRowsToSeries(options.rows ?? []), + tag: options.tag, from: options.from, to: options.to, - resolutionSeconds: options.resolutionSeconds, + granulate: options.granulate, + series: options.rows.length + ? [ + { + edge: options.edge, + tag: options.tag, + points: mapHistoryRowsToPoints(options.rows), + }, + ] + : [], }; } -/** Группирует строки БД по edge/tag и переводит Date в миллисекунды Unix. */ -function mapHistoryRowsToSeries(rows: HistoryRow[]): HistorySeriesDto[] { - const seriesMap = new Map(); - - for (const row of rows) { - const key = `${row.edge}\u0000${row.tag}`; - const series = seriesMap.get(key) ?? { - edge: row.edge, - tag: row.tag, - points: [], - }; - - series.points.push({ - t: row.time.getTime(), - v: row.value, - min: row.min_value, - avg: row.avg_value, - max: row.max_value, - count: row.point_count, - }); - seriesMap.set(key, series); - } - - return Array.from(seriesMap.values()); +function mapHistoryRowsToPoints(rows: HistoryRow[]): HistorySeriesDto['points'] { + return rows.map((row) => ({ + t: row.time.getTime(), + min: row.min_value, + avg: row.avg_value, + max: row.max_value, + count: row.point_count, + })); } diff --git a/src/history/history.repository.ts b/src/history/history.repository.ts index 7afa2c0..1fee3e1 100644 --- a/src/history/history.repository.ts +++ b/src/history/history.repository.ts @@ -1,160 +1,45 @@ import { Injectable } from '@nestjs/common'; import { DbService } from '../db/db.service'; -import { HistoryRow, HistorySource } from './history.types'; +import { HistoryRow } from './history.types'; @Injectable() export class HistoryRepository { constructor(private readonly db: DbService) {} - /** Загружает последние N сырых точек истории для каждого запрошенного тега. */ - async findLatest(edge: string, tags: string[], targetPoints: number): Promise { - const result = await this.db.query( - ` - SELECT - latest.edge, - latest.tag, - latest.timestamp AS time, - latest.value, - latest.value AS min_value, - latest.value AS avg_value, - latest.value AS max_value, - 1::integer AS point_count - FROM unnest($2::varchar[]) AS requested(tag) - CROSS JOIN LATERAL ( - SELECT edge, tag, timestamp, value - FROM history - WHERE edge = $1 - AND tag = requested.tag - ORDER BY timestamp DESC - LIMIT $3 - ) AS latest - ORDER BY latest.tag ASC, latest.timestamp ASC - `, - [edge, tags, targetPoints], - ); - - return result.rows; - } - - /** Загружает сырую историю за диапазон и агрегирует ее в бакеты под бюджет точек. */ - async findRawRange( + /** Агрегирует сырую историю по одному тегу через TimescaleDB time_bucket. */ + async findBucketedRange( edge: string, - tags: string[], + tag: string, from: Date, to: Date, - targetPoints: number, + granulate: string, ): Promise { - const bucketMillis = this.getBucketMillis(from, to, targetPoints); - const fromMillis = from.getTime(); const result = await this.db.query( ` - WITH filtered AS ( - SELECT - edge, - tag, - floor((extract(epoch FROM timestamp) * 1000 - $5) / $6)::bigint AS bucket_id, - value - FROM history - WHERE edge = $1 - AND tag = ANY($2::varchar[]) - AND timestamp >= $3 - AND timestamp <= $4 - ) SELECT - edge, - tag, - to_timestamp(($5 + bucket_id * $6) / 1000.0) AS time, - avg(value)::double precision AS value, + $1::varchar AS edge, + $2::varchar AS tag, + bucket AS time, min(value)::double precision AS min_value, avg(value)::double precision AS avg_value, max(value)::double precision AS max_value, count(*)::integer AS point_count - FROM filtered - GROUP BY edge, tag, bucket_id - ORDER BY tag ASC, time ASC - `, - [edge, tags, from, to, fromMillis, bucketMillis], - ); - - return result.rows; - } - - /** Читает заранее подготовленный aggregate-view с min/avg/max/count по бакетам. */ - async findAggregateRange( - edge: string, - tags: string[], - from: Date, - to: Date, - source: HistorySource, - ): Promise { - const result = await this.db.query( - ` - SELECT - edge, - tag, - ${source.timeColumn} AS time, - avg_value AS value, - min_value, - avg_value, - max_value, - point_count - FROM ${source.table} - WHERE edge = $1 - AND tag = ANY($2::varchar[]) - AND ${source.timeColumn} >= $3 - AND ${source.timeColumn} <= $4 - ORDER BY tag ASC, ${source.timeColumn} ASC - `, - [edge, tags, from, to], - ); - - return result.rows; - } - - /** Рассчитывает ширину временного бакета так, чтобы на тег было не больше targetPoints точек. */ - private getBucketMillis(from: Date, to: Date, targetPoints: number): number { - const durationMillis = Math.max(to.getTime() - from.getTime(), 1); - return Math.max(Math.ceil(durationMillis / Math.max(targetPoints, 1)), 1); - } - - /** Берет теги из запроса или ищет связи edge, а затем резервно смотрит саму историю. */ - async resolveTags(edge: string, requestedTags?: string[]): Promise { - if (requestedTags?.length) { - return requestedTags; - } - - const linked = await this.db.query<{ tag: string }>( - ` - SELECT DISTINCT tag FROM ( - SELECT unnest(e.tag_ids)::text AS tag - FROM edge AS e - WHERE e.id = $1 - UNION - SELECT t.id AS tag - FROM tag AS t - WHERE $1 = ANY(t.edge_ids) - ) AS linked_tags - WHERE tag IS NOT NULL - ORDER BY tag ASC + SELECT + time_bucket($5::interval, timestamp) AS bucket, + value + FROM history + WHERE edge = $1 + AND tag = $2 + AND timestamp >= $3 + AND timestamp < $4 + ) AS bucketed + GROUP BY bucket + ORDER BY bucket ASC `, - [edge], + [edge, tag, from, to, granulate], ); - if (linked.rows.length) { - return linked.rows.map((row) => row.tag); - } - - const distinct = await this.db.query<{ tag: string }>( - ` - SELECT DISTINCT tag - FROM history - WHERE edge = $1 - ORDER BY tag ASC - `, - [edge], - ); - - return distinct.rows.map((row) => row.tag); + return result.rows; } } diff --git a/src/history/history.service.ts b/src/history/history.service.ts index 4bf6799..8e80f85 100644 --- a/src/history/history.service.ts +++ b/src/history/history.service.ts @@ -1,66 +1,17 @@ import { BadRequestException, Injectable } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { getIntegerConfig } from '../common/config-number'; -import { normalizeRequiredText } from '../common/normalize-text'; import { GetHistoryDto } from './dto/get-history.dto'; import { HistoryResponseDto } from './dto/history-response.dto'; -import { chooseHistorySource } from './history-source'; import { createHistoryResponse } from './history.mapper'; import { HistoryRepository } from './history.repository'; @Injectable() export class HistoryService { - constructor( - private readonly repository: HistoryRepository, - private readonly config: ConfigService, - ) {} + constructor(private readonly repository: HistoryRepository) {} - /** Определяет теги, выбирает слой истории и собирает API-ответ для графика. */ + /** Получает агрегированную историю одного тега за диапазон с заданным шагом bucket. */ async findSeries(query: GetHistoryDto): Promise { - const edge = normalizeRequiredText(query.edge, 'edge'); - const targetPoints = this.getTargetPoints(query.targetPoints); - const tags = await this.repository.resolveTags(edge, query.tags); - - if (!tags.length) { - return createHistoryResponse({ - edge, - source: 'empty', - targetPoints, - }); - } - - if (!query.from && !query.to) { - const rows = await this.repository.findLatest(edge, tags, targetPoints); - return createHistoryResponse({ - edge, - source: 'latest', - targetPoints, - rows, - }); - } - - const { from, to } = this.parseRange(query); - const source = chooseHistorySource(from, to, targetPoints); - const rows = - source.name === 'raw' - ? await this.repository.findRawRange(edge, tags, from, to, targetPoints) - : await this.repository.findAggregateRange(edge, tags, from, to, source); - - return createHistoryResponse({ - edge, - from, - to, - source: source.name, - resolutionSeconds: source.intervalSeconds || null, - targetPoints, - rows, - }); - } - - /** Разбирает опциональный диапазон времени и отклоняет неверные или обратные границы. */ - private parseRange(query: GetHistoryDto): { from: Date; to: Date } { - const from = query.from ? new Date(query.from) : new Date(0); - const to = query.to ? new Date(query.to) : new Date(); + const from = new Date(query.from); + const to = new Date(query.to); if (Number.isNaN(from.getTime()) || Number.isNaN(to.getTime())) { throw new BadRequestException('Invalid from/to date.'); @@ -70,14 +21,21 @@ export class HistoryService { throw new BadRequestException('from cannot be greater than to.'); } - return { from, to }; - } + const rows = await this.repository.findBucketedRange( + query.edge, + query.tag, + from, + to, + query.granulate, + ); - /** Применяет лимиты точек по умолчанию и максимуму, чтобы ответ истории был удобным для графиков. */ - private getTargetPoints(requested?: number): number { - const defaultValue = getIntegerConfig(this.config, 'HISTORY_TARGET_POINTS', 2000); - const maxValue = getIntegerConfig(this.config, 'HISTORY_MAX_TARGET_POINTS', 5000); - const value = requested ?? defaultValue; - return Math.min(Math.max(Math.floor(value), 100), maxValue); + return createHistoryResponse({ + edge: query.edge, + tag: query.tag, + from, + to, + granulate: query.granulate, + rows, + }); } } diff --git a/src/history/history.types.ts b/src/history/history.types.ts index 246320d..470112b 100644 --- a/src/history/history.types.ts +++ b/src/history/history.types.ts @@ -1,17 +1,7 @@ -export type HistorySourceName = 'raw' | '1m' | '5m' | '1h' | '1d'; - -export type HistorySource = { - name: HistorySourceName; - table: string; - timeColumn: string; - intervalSeconds: number; -}; - export type HistoryRow = { edge: string; tag: string; time: Date; - value: number; min_value: number; avg_value: number; max_value: number; diff --git a/src/ingest/ingest.service.ts b/src/ingest/ingest.service.ts index a6d2a37..dd575f4 100644 --- a/src/ingest/ingest.service.ts +++ b/src/ingest/ingest.service.ts @@ -1,12 +1,7 @@ import { BadRequestException, Injectable } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { getIntegerConfig } from '../common/config-number'; -import { normalizeRequiredText } from '../common/normalize-text'; import { DbService } from '../db/db.service'; import { IngestPointDto } from './dto/ingest-point.dto'; -const DEFAULT_MAX_BATCH_SIZE = 10_000; - type NormalizedPoint = { time: Date; edge: string; @@ -20,10 +15,7 @@ type IngestResult = { @Injectable() export class IngestService { - constructor( - private readonly db: DbService, - private readonly config: ConfigService, - ) {} + constructor(private readonly db: DbService) {} /** Проверяет батч, пишет сырую историю и обновляет последний снимок current для UI/SSE. */ async ingestPoints(points: IngestPointDto[]): Promise { @@ -89,10 +81,7 @@ export class IngestService { /** Защищает API от слишком больших ingest-запросов, занимающих соединение с БД надолго. */ private assertBatchSize(size: number): void { - const maxBatchSize = Math.max( - getIntegerConfig(this.config, 'INGEST_MAX_BATCH_SIZE', DEFAULT_MAX_BATCH_SIZE), - 1, - ); + const maxBatchSize = Number(process.env.INGEST_MAX_BATCH_SIZE); if (size > maxBatchSize) { throw new BadRequestException(`Batch size must not exceed ${maxBatchSize} points.`); @@ -101,17 +90,11 @@ export class IngestService { /** Приводит входной DTO к нормализованному виду для вставки в history. */ private normalizePoint(point: IngestPointDto): NormalizedPoint { - const edge = normalizeRequiredText(point.edge, 'edge'); - const tag = normalizeRequiredText(point.tag, 'tag'); const time = this.parsePointTime(point); - if (!Number.isFinite(point.value)) { - throw new BadRequestException('value must be a finite number.'); - } - return { - edge, - tag, + edge: point.edge, + tag: point.tag, value: point.value, time, }; diff --git a/src/main.ts b/src/main.ts index 251a044..f190323 100644 --- a/src/main.ts +++ b/src/main.ts @@ -8,12 +8,8 @@ async function bootstrap() { const app = await NestFactory.create(AppModule); app.use(compression()); - const corsAllowedOrigins = process.env.CORS_ALLOWED_ORIGINS - ? process.env.CORS_ALLOWED_ORIGINS.split(',').map((origin) => origin.trim()).filter(Boolean) - : true; - app.enableCors({ - origin: corsAllowedOrigins, + origin: process.env.CORS_ALLOWED_ORIGINS, credentials: true, methods: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], allowedHeaders: ['Content-Type', 'Authorization', 'x-api-key'], @@ -27,7 +23,7 @@ async function bootstrap() { }), ); - await app.listen(process.env.PORT ?? 3101); + await app.listen(process.env.PORT as string); } void bootstrap(); diff --git a/src/tag/dto/get-tags.dto.ts b/src/tag/dto/get-tags.dto.ts index f484845..ae8e5b6 100644 --- a/src/tag/dto/get-tags.dto.ts +++ b/src/tag/dto/get-tags.dto.ts @@ -1,15 +1,11 @@ -import { Transform } from 'class-transformer'; import { IsOptional, IsString } from 'class-validator'; -import { normalizeOptionalText } from '../../common/normalize-text'; export class GetTagsDto { @IsOptional() - @Transform(({ value }) => normalizeOptionalText(value)) @IsString() edge?: string; @IsOptional() - @Transform(({ value }) => normalizeOptionalText(value)) @IsString() search?: string; }