Обновил прием данных из mqtt-ingest на поэлементный. Убрал batch сохранение. Скорректировал update в таблицу current.

This commit is contained in:
Первов Артем
2026-06-22 02:50:24 +03:00
parent ce493f8fb6
commit 05c13f36a4
3 changed files with 36 additions and 62 deletions

View File

@@ -7,7 +7,7 @@ Lightweight Drill Cloud API for an existing PostgreSQL/TimescaleDB database.
- No migrations. - No migrations.
- No `tag-translation` module. - No `tag-translation` module.
- No alarm log writes. - No alarm log writes.
- No automatic `current`, `edge`, or `tag` upserts during ingest. - No automatic `edge` or `tag` upserts during ingest.
## Database ## Database
@@ -36,17 +36,15 @@ Historical chart ranges use continuous aggregates:
- `POST /ingest` - `POST /ingest`
- `POST /ingest/:edge` - `POST /ingest/:edge`
`POST /ingest` accepts an array of points: `POST /ingest` accepts one point:
```json ```json
[
{ {
"edge": "edge5", "edge": "edge5",
"tag": "hook_weight_1", "tag": "hook_weight_1",
"time": "2026-06-19T00:00:00Z", "time": "2026-06-19T00:00:00Z",
"value": 12.34 "value": 12.34
} }
]
``` ```
`POST /ingest/:edge` accepts a compact edge payload: `POST /ingest/:edge` accepts a compact edge payload:

View File

@@ -1,4 +1,4 @@
import { Body, Controller, Param, ParseArrayPipe, Post, UseGuards } from '@nestjs/common'; import { Body, Controller, Param, Post, UseGuards } from '@nestjs/common';
import { IngestApiKeyGuard } from '../common/ingest-api-key.guard'; import { IngestApiKeyGuard } from '../common/ingest-api-key.guard';
import { IngestEdgeValuesDto } from './dto/ingest-edge-values.dto'; import { IngestEdgeValuesDto } from './dto/ingest-edge-values.dto';
import { IngestPointDto } from './dto/ingest-point.dto'; import { IngestPointDto } from './dto/ingest-point.dto';
@@ -9,16 +9,13 @@ import { IngestService } from './ingest.service';
export class IngestController { export class IngestController {
constructor(private readonly ingest: IngestService) {} constructor(private readonly ingest: IngestService) {}
/** Принимает массив сырых точек и сохраняет их как записи истории. */ /** Принимает одну сырую точку и сохраняет ее как запись истории/current. */
@Post() @Post()
ingestPoints( ingestPoint(@Body() point: IngestPointDto) {
@Body(new ParseArrayPipe({ items: IngestPointDto })) return this.ingest.ingestPoint(point);
points: IngestPointDto[],
) {
return this.ingest.ingestPoints(points);
} }
/** Принимает компактное тело запроса по edge и разворачивает каждое значение в точку истории. */ /** Принимает компактное тело запроса по edge и сохраняет каждое значение построчно. */
@Post(':edge') @Post(':edge')
ingestEdgeValues(@Param('edge') edge: string, @Body() body: IngestEdgeValuesDto) { ingestEdgeValues(@Param('edge') edge: string, @Body() body: IngestEdgeValuesDto) {
const points = Object.entries(body.values).map(([tag, value]) => ({ const points = Object.entries(body.values).map(([tag, value]) => ({

View File

@@ -17,68 +17,47 @@ type IngestResult = {
export class IngestService { export class IngestService {
constructor(private readonly db: DbService) {} constructor(private readonly db: DbService) {}
/** Проверяет батч, пишет сырую историю и обновляет последний снимок current для UI/SSE. */ /** Пишет одну сырую точку в history и обновляет текущий снимок current для UI/SSE. */
async ingestPoints(points: IngestPointDto[]): Promise<IngestResult> { async ingestPoint(point: IngestPointDto): Promise<IngestResult> {
if (!points.length) { const normalized: NormalizedPoint = {
return { processed: 0 };
}
const normalized: NormalizedPoint[] = points.map((point) => ({
edge: point.edge, edge: point.edge,
tag: point.tag, tag: point.tag,
value: point.value, value: point.value,
time: new Date((point.time ?? point.timestamp) as string | number), time: new Date((point.time ?? point.timestamp) as string | number),
})); };
await this.db.query( await this.db.query(
` `
INSERT INTO history (timestamp, edge, tag, value) INSERT INTO history (timestamp, edge, tag, value)
SELECT time, edge, tag, value VALUES ($1, $2, $3, $4)
FROM unnest(
$1::timestamp[],
$2::varchar[],
$3::varchar[],
$4::double precision[]
) AS point(time, edge, tag, value)
`, `,
[ [normalized.time, normalized.edge, normalized.tag, normalized.value],
normalized.map((point) => point.time),
normalized.map((point) => point.edge),
normalized.map((point) => point.tag),
normalized.map((point) => point.value),
],
); );
await this.db.query( await this.db.query(
` `
INSERT INTO current (edge, tag, value, "createdAt", "updatedAt") INSERT INTO current (edge, tag, value, "createdAt", "updatedAt")
SELECT DISTINCT ON (edge, tag) VALUES ($1, $2, $3, NOW(), NOW())
edge,
tag,
value,
time AS "createdAt",
time AS "updatedAt"
FROM unnest(
$1::timestamp[],
$2::varchar[],
$3::varchar[],
$4::double precision[]
) AS point(time, edge, tag, value)
ORDER BY edge ASC, tag ASC, time DESC
ON CONFLICT (edge, tag) ON CONFLICT (edge, tag)
DO UPDATE SET DO UPDATE SET
value = EXCLUDED.value, value = $3,
"updatedAt" = EXCLUDED."updatedAt" "updatedAt" = NOW()
WHERE current."updatedAt" <= EXCLUDED."updatedAt"
`, `,
[ [normalized.edge, normalized.tag, normalized.value],
normalized.map((point) => point.time),
normalized.map((point) => point.edge),
normalized.map((point) => point.tag),
normalized.map((point) => point.value),
],
); );
return { processed: normalized.length }; return { processed: 1 };
}
/** Сохраняет несколько точек без batch SQL: каждая точка проходит тот же линейный путь. */
async ingestPoints(points: IngestPointDto[]): Promise<IngestResult> {
let processed = 0;
for (const point of points) {
const result = await this.ingestPoint(point);
processed += result.processed;
}
return { processed };
} }
} }