Обновил прием данных из mqtt-ingest на поэлементный. Убрал batch сохранение. Скорректировал update в таблицу current.
This commit is contained in:
18
README.md
18
README.md
@@ -7,7 +7,7 @@ Lightweight Drill Cloud API for an existing PostgreSQL/TimescaleDB database.
|
||||
- No migrations.
|
||||
- No `tag-translation` module.
|
||||
- No alarm log writes.
|
||||
- No automatic `current`, `edge`, or `tag` upserts during ingest.
|
||||
- No automatic `edge` or `tag` upserts during ingest.
|
||||
|
||||
## Database
|
||||
|
||||
@@ -36,17 +36,15 @@ Historical chart ranges use continuous aggregates:
|
||||
- `POST /ingest`
|
||||
- `POST /ingest/:edge`
|
||||
|
||||
`POST /ingest` accepts an array of points:
|
||||
`POST /ingest` accepts one point:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"edge": "edge5",
|
||||
"tag": "hook_weight_1",
|
||||
"time": "2026-06-19T00:00:00Z",
|
||||
"value": 12.34
|
||||
}
|
||||
]
|
||||
{
|
||||
"edge": "edge5",
|
||||
"tag": "hook_weight_1",
|
||||
"time": "2026-06-19T00:00:00Z",
|
||||
"value": 12.34
|
||||
}
|
||||
```
|
||||
|
||||
`POST /ingest/:edge` accepts a compact edge payload:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Body, Controller, Param, ParseArrayPipe, Post, UseGuards } from '@nestjs/common';
|
||||
import { Body, Controller, Param, Post, UseGuards } from '@nestjs/common';
|
||||
import { IngestApiKeyGuard } from '../common/ingest-api-key.guard';
|
||||
import { IngestEdgeValuesDto } from './dto/ingest-edge-values.dto';
|
||||
import { IngestPointDto } from './dto/ingest-point.dto';
|
||||
@@ -9,16 +9,13 @@ import { IngestService } from './ingest.service';
|
||||
export class IngestController {
|
||||
constructor(private readonly ingest: IngestService) {}
|
||||
|
||||
/** Принимает массив сырых точек и сохраняет их как записи истории. */
|
||||
/** Принимает одну сырую точку и сохраняет ее как запись истории/current. */
|
||||
@Post()
|
||||
ingestPoints(
|
||||
@Body(new ParseArrayPipe({ items: IngestPointDto }))
|
||||
points: IngestPointDto[],
|
||||
) {
|
||||
return this.ingest.ingestPoints(points);
|
||||
ingestPoint(@Body() point: IngestPointDto) {
|
||||
return this.ingest.ingestPoint(point);
|
||||
}
|
||||
|
||||
/** Принимает компактное тело запроса по edge и разворачивает каждое значение в точку истории. */
|
||||
/** Принимает компактное тело запроса по edge и сохраняет каждое значение построчно. */
|
||||
@Post(':edge')
|
||||
ingestEdgeValues(@Param('edge') edge: string, @Body() body: IngestEdgeValuesDto) {
|
||||
const points = Object.entries(body.values).map(([tag, value]) => ({
|
||||
|
||||
@@ -17,68 +17,47 @@ type IngestResult = {
|
||||
export class IngestService {
|
||||
constructor(private readonly db: DbService) {}
|
||||
|
||||
/** Проверяет батч, пишет сырую историю и обновляет последний снимок current для UI/SSE. */
|
||||
async ingestPoints(points: IngestPointDto[]): Promise<IngestResult> {
|
||||
if (!points.length) {
|
||||
return { processed: 0 };
|
||||
}
|
||||
|
||||
const normalized: NormalizedPoint[] = points.map((point) => ({
|
||||
/** Пишет одну сырую точку в history и обновляет текущий снимок current для UI/SSE. */
|
||||
async ingestPoint(point: IngestPointDto): Promise<IngestResult> {
|
||||
const normalized: NormalizedPoint = {
|
||||
edge: point.edge,
|
||||
tag: point.tag,
|
||||
value: point.value,
|
||||
time: new Date((point.time ?? point.timestamp) as string | number),
|
||||
}));
|
||||
};
|
||||
|
||||
await this.db.query(
|
||||
`
|
||||
INSERT INTO history (timestamp, edge, tag, value)
|
||||
SELECT time, edge, tag, value
|
||||
FROM unnest(
|
||||
$1::timestamp[],
|
||||
$2::varchar[],
|
||||
$3::varchar[],
|
||||
$4::double precision[]
|
||||
) AS point(time, edge, tag, value)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
`,
|
||||
[
|
||||
normalized.map((point) => point.time),
|
||||
normalized.map((point) => point.edge),
|
||||
normalized.map((point) => point.tag),
|
||||
normalized.map((point) => point.value),
|
||||
],
|
||||
[normalized.time, normalized.edge, normalized.tag, normalized.value],
|
||||
);
|
||||
|
||||
await this.db.query(
|
||||
`
|
||||
INSERT INTO current (edge, tag, value, "createdAt", "updatedAt")
|
||||
SELECT DISTINCT ON (edge, tag)
|
||||
edge,
|
||||
tag,
|
||||
value,
|
||||
time AS "createdAt",
|
||||
time AS "updatedAt"
|
||||
FROM unnest(
|
||||
$1::timestamp[],
|
||||
$2::varchar[],
|
||||
$3::varchar[],
|
||||
$4::double precision[]
|
||||
) AS point(time, edge, tag, value)
|
||||
ORDER BY edge ASC, tag ASC, time DESC
|
||||
VALUES ($1, $2, $3, NOW(), NOW())
|
||||
ON CONFLICT (edge, tag)
|
||||
DO UPDATE SET
|
||||
value = EXCLUDED.value,
|
||||
"updatedAt" = EXCLUDED."updatedAt"
|
||||
WHERE current."updatedAt" <= EXCLUDED."updatedAt"
|
||||
value = $3,
|
||||
"updatedAt" = NOW()
|
||||
`,
|
||||
[
|
||||
normalized.map((point) => point.time),
|
||||
normalized.map((point) => point.edge),
|
||||
normalized.map((point) => point.tag),
|
||||
normalized.map((point) => point.value),
|
||||
],
|
||||
[normalized.edge, normalized.tag, normalized.value],
|
||||
);
|
||||
|
||||
return { processed: normalized.length };
|
||||
return { processed: 1 };
|
||||
}
|
||||
|
||||
/** Сохраняет несколько точек без batch SQL: каждая точка проходит тот же линейный путь. */
|
||||
async ingestPoints(points: IngestPointDto[]): Promise<IngestResult> {
|
||||
let processed = 0;
|
||||
|
||||
for (const point of points) {
|
||||
const result = await this.ingestPoint(point);
|
||||
processed += result.processed;
|
||||
}
|
||||
|
||||
return { processed };
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user