Обновил прием данных из mqtt-ingest на поэлементный. Убрал batch сохранение. Скорректировал update в таблицу current.
This commit is contained in:
18
README.md
18
README.md
@@ -7,7 +7,7 @@ Lightweight Drill Cloud API for an existing PostgreSQL/TimescaleDB database.
|
|||||||
- No migrations.
|
- No migrations.
|
||||||
- No `tag-translation` module.
|
- No `tag-translation` module.
|
||||||
- No alarm log writes.
|
- No alarm log writes.
|
||||||
- No automatic `current`, `edge`, or `tag` upserts during ingest.
|
- No automatic `edge` or `tag` upserts during ingest.
|
||||||
|
|
||||||
## Database
|
## Database
|
||||||
|
|
||||||
@@ -36,17 +36,15 @@ Historical chart ranges use continuous aggregates:
|
|||||||
- `POST /ingest`
|
- `POST /ingest`
|
||||||
- `POST /ingest/:edge`
|
- `POST /ingest/:edge`
|
||||||
|
|
||||||
`POST /ingest` accepts an array of points:
|
`POST /ingest` accepts one point:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
[
|
{
|
||||||
{
|
"edge": "edge5",
|
||||||
"edge": "edge5",
|
"tag": "hook_weight_1",
|
||||||
"tag": "hook_weight_1",
|
"time": "2026-06-19T00:00:00Z",
|
||||||
"time": "2026-06-19T00:00:00Z",
|
"value": 12.34
|
||||||
"value": 12.34
|
}
|
||||||
}
|
|
||||||
]
|
|
||||||
```
|
```
|
||||||
|
|
||||||
`POST /ingest/:edge` accepts a compact edge payload:
|
`POST /ingest/:edge` accepts a compact edge payload:
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { Body, Controller, Param, ParseArrayPipe, Post, UseGuards } from '@nestjs/common';
|
import { Body, Controller, Param, Post, UseGuards } from '@nestjs/common';
|
||||||
import { IngestApiKeyGuard } from '../common/ingest-api-key.guard';
|
import { IngestApiKeyGuard } from '../common/ingest-api-key.guard';
|
||||||
import { IngestEdgeValuesDto } from './dto/ingest-edge-values.dto';
|
import { IngestEdgeValuesDto } from './dto/ingest-edge-values.dto';
|
||||||
import { IngestPointDto } from './dto/ingest-point.dto';
|
import { IngestPointDto } from './dto/ingest-point.dto';
|
||||||
@@ -9,16 +9,13 @@ import { IngestService } from './ingest.service';
|
|||||||
export class IngestController {
|
export class IngestController {
|
||||||
constructor(private readonly ingest: IngestService) {}
|
constructor(private readonly ingest: IngestService) {}
|
||||||
|
|
||||||
/** Принимает массив сырых точек и сохраняет их как записи истории. */
|
/** Принимает одну сырую точку и сохраняет ее как запись истории/current. */
|
||||||
@Post()
|
@Post()
|
||||||
ingestPoints(
|
ingestPoint(@Body() point: IngestPointDto) {
|
||||||
@Body(new ParseArrayPipe({ items: IngestPointDto }))
|
return this.ingest.ingestPoint(point);
|
||||||
points: IngestPointDto[],
|
|
||||||
) {
|
|
||||||
return this.ingest.ingestPoints(points);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Принимает компактное тело запроса по edge и разворачивает каждое значение в точку истории. */
|
/** Принимает компактное тело запроса по edge и сохраняет каждое значение построчно. */
|
||||||
@Post(':edge')
|
@Post(':edge')
|
||||||
ingestEdgeValues(@Param('edge') edge: string, @Body() body: IngestEdgeValuesDto) {
|
ingestEdgeValues(@Param('edge') edge: string, @Body() body: IngestEdgeValuesDto) {
|
||||||
const points = Object.entries(body.values).map(([tag, value]) => ({
|
const points = Object.entries(body.values).map(([tag, value]) => ({
|
||||||
|
|||||||
@@ -17,68 +17,47 @@ type IngestResult = {
|
|||||||
export class IngestService {
|
export class IngestService {
|
||||||
constructor(private readonly db: DbService) {}
|
constructor(private readonly db: DbService) {}
|
||||||
|
|
||||||
/** Проверяет батч, пишет сырую историю и обновляет последний снимок current для UI/SSE. */
|
/** Пишет одну сырую точку в history и обновляет текущий снимок current для UI/SSE. */
|
||||||
async ingestPoints(points: IngestPointDto[]): Promise<IngestResult> {
|
async ingestPoint(point: IngestPointDto): Promise<IngestResult> {
|
||||||
if (!points.length) {
|
const normalized: NormalizedPoint = {
|
||||||
return { processed: 0 };
|
|
||||||
}
|
|
||||||
|
|
||||||
const normalized: NormalizedPoint[] = points.map((point) => ({
|
|
||||||
edge: point.edge,
|
edge: point.edge,
|
||||||
tag: point.tag,
|
tag: point.tag,
|
||||||
value: point.value,
|
value: point.value,
|
||||||
time: new Date((point.time ?? point.timestamp) as string | number),
|
time: new Date((point.time ?? point.timestamp) as string | number),
|
||||||
}));
|
};
|
||||||
|
|
||||||
await this.db.query(
|
await this.db.query(
|
||||||
`
|
`
|
||||||
INSERT INTO history (timestamp, edge, tag, value)
|
INSERT INTO history (timestamp, edge, tag, value)
|
||||||
SELECT time, edge, tag, value
|
VALUES ($1, $2, $3, $4)
|
||||||
FROM unnest(
|
|
||||||
$1::timestamp[],
|
|
||||||
$2::varchar[],
|
|
||||||
$3::varchar[],
|
|
||||||
$4::double precision[]
|
|
||||||
) AS point(time, edge, tag, value)
|
|
||||||
`,
|
`,
|
||||||
[
|
[normalized.time, normalized.edge, normalized.tag, normalized.value],
|
||||||
normalized.map((point) => point.time),
|
|
||||||
normalized.map((point) => point.edge),
|
|
||||||
normalized.map((point) => point.tag),
|
|
||||||
normalized.map((point) => point.value),
|
|
||||||
],
|
|
||||||
);
|
);
|
||||||
|
|
||||||
await this.db.query(
|
await this.db.query(
|
||||||
`
|
`
|
||||||
INSERT INTO current (edge, tag, value, "createdAt", "updatedAt")
|
INSERT INTO current (edge, tag, value, "createdAt", "updatedAt")
|
||||||
SELECT DISTINCT ON (edge, tag)
|
VALUES ($1, $2, $3, NOW(), NOW())
|
||||||
edge,
|
|
||||||
tag,
|
|
||||||
value,
|
|
||||||
time AS "createdAt",
|
|
||||||
time AS "updatedAt"
|
|
||||||
FROM unnest(
|
|
||||||
$1::timestamp[],
|
|
||||||
$2::varchar[],
|
|
||||||
$3::varchar[],
|
|
||||||
$4::double precision[]
|
|
||||||
) AS point(time, edge, tag, value)
|
|
||||||
ORDER BY edge ASC, tag ASC, time DESC
|
|
||||||
ON CONFLICT (edge, tag)
|
ON CONFLICT (edge, tag)
|
||||||
DO UPDATE SET
|
DO UPDATE SET
|
||||||
value = EXCLUDED.value,
|
value = $3,
|
||||||
"updatedAt" = EXCLUDED."updatedAt"
|
"updatedAt" = NOW()
|
||||||
WHERE current."updatedAt" <= EXCLUDED."updatedAt"
|
|
||||||
`,
|
`,
|
||||||
[
|
[normalized.edge, normalized.tag, normalized.value],
|
||||||
normalized.map((point) => point.time),
|
|
||||||
normalized.map((point) => point.edge),
|
|
||||||
normalized.map((point) => point.tag),
|
|
||||||
normalized.map((point) => point.value),
|
|
||||||
],
|
|
||||||
);
|
);
|
||||||
|
|
||||||
return { processed: normalized.length };
|
return { processed: 1 };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Сохраняет несколько точек без batch SQL: каждая точка проходит тот же линейный путь. */
|
||||||
|
async ingestPoints(points: IngestPointDto[]): Promise<IngestResult> {
|
||||||
|
let processed = 0;
|
||||||
|
|
||||||
|
for (const point of points) {
|
||||||
|
const result = await this.ingestPoint(point);
|
||||||
|
processed += result.processed;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { processed };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user