u
This commit is contained in:
@@ -159,6 +159,7 @@ Generated from `@openapi` annotations in [`src/api/routes/`](../../src/api/route
|
|||||||
| [`routes/entity/update.md`](routes/entity/update.md) | `PATCH /entity/{id}` |
|
| [`routes/entity/update.md`](routes/entity/update.md) | `PATCH /entity/{id}` |
|
||||||
| [`routes/events.md`](routes/events.md) | `GET /events` |
|
| [`routes/events.md`](routes/events.md) | `GET /events` |
|
||||||
| [`routes/health.md`](routes/health.md) | `GET /health` |
|
| [`routes/health.md`](routes/health.md) | `GET /health` |
|
||||||
|
| [`routes/health/latency.md`](routes/health/latency.md) | `GET /health/latency` |
|
||||||
| [`routes/image/get.md`](routes/image/get.md) | `GET /image/{fullHash}` |
|
| [`routes/image/get.md`](routes/image/get.md) | `GET /image/{fullHash}` |
|
||||||
| [`routes/labeltemplate/list.md`](routes/labeltemplate/list.md) | `GET /labeltemplate` |
|
| [`routes/labeltemplate/list.md`](routes/labeltemplate/list.md) | `GET /labeltemplate` |
|
||||||
| [`routes/labeltemplate/create.md`](routes/labeltemplate/create.md) | `POST /labeltemplate` |
|
| [`routes/labeltemplate/create.md`](routes/labeltemplate/create.md) | `POST /labeltemplate` |
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Dash API Routes Overview (All)
|
# Dash API Routes Overview (All)
|
||||||
|
|
||||||
Overview of all 263 API endpoints across all modules.
|
Overview of all 264 API endpoints across all modules.
|
||||||
|
|
||||||
### Notation
|
### Notation
|
||||||
|
|
||||||
@@ -498,6 +498,9 @@ _none_
|
|||||||
#### Database health check — `GET /health`
|
#### Database health check — `GET /health`
|
||||||
_none_
|
_none_
|
||||||
- Returns: `200 { ok: bool }`
|
- Returns: `200 { ok: bool }`
|
||||||
|
#### Health latency history — `GET /health/latency`
|
||||||
|
_none_
|
||||||
|
- Returns: `200 { raw: { timestamp: int, value: num }[], avg: { timestamp: int, value: num }[], max: { timestamp: int, value: num }[] }`
|
||||||
|
|
||||||
### Section: /image
|
### Section: /image
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Dash API Routes Overview
|
# Dash API Routes Overview
|
||||||
|
|
||||||
Overview of all 186 API endpoints across all modules.
|
Overview of all 187 API endpoints across all modules.
|
||||||
|
|
||||||
### Notation
|
### Notation
|
||||||
|
|
||||||
@@ -381,6 +381,12 @@ _none_
|
|||||||
- Params: `id: int, body: { name?: str, firstName?: str|null, number?: str|null }`
|
- Params: `id: int, body: { name?: str, firstName?: str|null, number?: str|null }`
|
||||||
- Returns: `200 { entity: T19 }`
|
- Returns: `200 { entity: T19 }`
|
||||||
|
|
||||||
|
### Section: /health
|
||||||
|
|
||||||
|
#### Health latency history — `GET /health/latency`
|
||||||
|
_none_
|
||||||
|
- Returns: `200 { raw: { timestamp: int, value: num }[], avg: { timestamp: int, value: num }[], max: { timestamp: int, value: num }[] }`
|
||||||
|
|
||||||
### Section: /labeltemplate
|
### Section: /labeltemplate
|
||||||
|
|
||||||
#### List label templates — `GET /labeltemplate`
|
#### List label templates — `GET /labeltemplate`
|
||||||
|
|||||||
45
docs/api/routes/health/latency.md
Normal file
45
docs/api/routes/health/latency.md
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
# `GET /health/latency`
|
||||||
|
|
||||||
|
Protected. Requires a valid session cookie (`credentials: 'include'`).
|
||||||
|
|
||||||
|
Health latency history
|
||||||
|
|
||||||
|
Returns the database healthcheck latency (ms) as three series, each oldest first with unix-ms timestamps. `raw` holds the per-minute samples of the last 3 hours; `avg` and `max` hold the 30-minute aggregates (over the last 4 hours of samples) kept for 30 days. Requires a valid JWT; no company selection needed.
|
||||||
|
|
||||||
|
## Response `200`
|
||||||
|
|
||||||
|
Latency series
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"raw": [
|
||||||
|
{
|
||||||
|
"timestamp": 1,
|
||||||
|
"value": 1
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"avg": [
|
||||||
|
{
|
||||||
|
"timestamp": 1,
|
||||||
|
"value": 1
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"max": [
|
||||||
|
{
|
||||||
|
"timestamp": 1,
|
||||||
|
"value": 1
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Errors
|
||||||
|
|
||||||
|
| Status | Description |
|
||||||
|
|--------|-------------|
|
||||||
|
| 401 | Unauthorized |
|
||||||
|
| 500 | Redis read failed |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
_Generated from `@openapi` JSDoc. Do not edit by hand — run `npm run docs:api`._
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import { getStore } from '../../db/store.js';
|
import { getStore } from '../../db/store.js';
|
||||||
|
import { getLatencySeries } from '../../services/healthMonitorService.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Registers a database connectivity health check route.
|
* Registers a database connectivity health check route.
|
||||||
@@ -57,4 +58,57 @@ export function register(app, basePath = '') {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// #endregion
|
// #endregion
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @openapi
|
||||||
|
* /health/latency:
|
||||||
|
* get:
|
||||||
|
* tags: [Health]
|
||||||
|
* summary: Health latency history
|
||||||
|
* description: Returns the database healthcheck latency (ms) as three series, each oldest first with unix-ms timestamps. `raw` holds the per-minute samples of the last 3 hours; `avg` and `max` hold the 30-minute aggregates (over the last 4 hours of samples) kept for 30 days. Requires a valid JWT; no company selection needed.
|
||||||
|
* responses:
|
||||||
|
* 200:
|
||||||
|
* description: Latency series
|
||||||
|
* content:
|
||||||
|
* application/json:
|
||||||
|
* schema:
|
||||||
|
* type: object
|
||||||
|
* properties:
|
||||||
|
* raw:
|
||||||
|
* type: array
|
||||||
|
* items: { $ref: '#/components/schemas/LatencyPoint' }
|
||||||
|
* avg:
|
||||||
|
* type: array
|
||||||
|
* items: { $ref: '#/components/schemas/LatencyPoint' }
|
||||||
|
* max:
|
||||||
|
* type: array
|
||||||
|
* items: { $ref: '#/components/schemas/LatencyPoint' }
|
||||||
|
* 401:
|
||||||
|
* description: Unauthorized
|
||||||
|
* 500:
|
||||||
|
* description: Redis read failed
|
||||||
|
* components:
|
||||||
|
* schemas:
|
||||||
|
* LatencyPoint:
|
||||||
|
* type: object
|
||||||
|
* properties:
|
||||||
|
* timestamp:
|
||||||
|
* type: integer
|
||||||
|
* description: Unix epoch milliseconds
|
||||||
|
* value:
|
||||||
|
* type: number
|
||||||
|
* description: Latency in milliseconds
|
||||||
|
*/
|
||||||
|
// #region GET /health/latency
|
||||||
|
|
||||||
|
app.get(`${basePath}/health/latency`, async (_request, response) => {
|
||||||
|
try {
|
||||||
|
response.json(await getLatencySeries());
|
||||||
|
} catch (error) {
|
||||||
|
console.error('GET /health/latency failed:', error);
|
||||||
|
response.status(500).json({ error: error.message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// #endregion
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,8 +8,19 @@ const healthLog = createLogger('health-monitor');
|
|||||||
|
|
||||||
let scheduledTimer = null;
|
let scheduledTimer = null;
|
||||||
let scheduledInterval = null;
|
let scheduledInterval = null;
|
||||||
|
let aggregateInterval = null;
|
||||||
let isServiceRunning = false;
|
let isServiceRunning = false;
|
||||||
let redisClient = null;
|
let redisClient = null;
|
||||||
|
|
||||||
|
const RAW_RETENTION_MS = 3 * 60 * 60 * 1000;
|
||||||
|
const AGGREGATE_WINDOW_MS = 4 * 60 * 60 * 1000;
|
||||||
|
const AGGREGATE_INTERVAL_MS = 30 * 60 * 1000;
|
||||||
|
const AGGREGATE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
// MARK: redis sorted-set keys (score = unix ms timestamp, member = "<ts>:<value>")
|
||||||
|
export const rawKey = () => `${env.redis.key}:raw`;
|
||||||
|
export const avgKey = () => `${env.redis.key}:avg`;
|
||||||
|
export const maxKey = () => `${env.redis.key}:max`;
|
||||||
// #endregion
|
// #endregion
|
||||||
|
|
||||||
// #region Redis Client Management
|
// #region Redis Client Management
|
||||||
@@ -86,16 +97,70 @@ export function performHealthSelftest() {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: runs one selftest cycle and writes duration to Redis
|
// MARK: returns raw (3h), avg and max (30d) latency series as [{ timestamp, value }] oldest first
|
||||||
|
export async function getLatencySeries() {
|
||||||
|
const redis = getRedisClient();
|
||||||
|
const parse = (members) =>
|
||||||
|
members.map((m) => {
|
||||||
|
const i = m.indexOf(':');
|
||||||
|
return { timestamp: Number(m.slice(0, i)), value: Number(m.slice(i + 1)) };
|
||||||
|
});
|
||||||
|
const [raw, avg, max] = await Promise.all([
|
||||||
|
redis.zrange(rawKey(), 0, -1),
|
||||||
|
redis.zrange(avgKey(), 0, -1),
|
||||||
|
redis.zrange(maxKey(), 0, -1),
|
||||||
|
]);
|
||||||
|
return { raw: parse(raw), avg: parse(avg), max: parse(max) };
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: stores one latency sample and trims samples older than 3 hours (round-robin window)
|
||||||
|
export async function recordLatency(durationMs, now = Date.now()) {
|
||||||
|
const redis = getRedisClient();
|
||||||
|
await redis
|
||||||
|
.multi()
|
||||||
|
.zadd(rawKey(), now, `${now}:${durationMs}`)
|
||||||
|
.zremrangebyscore(rawKey(), '-inf', now - RAW_RETENTION_MS)
|
||||||
|
.exec();
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: computes avg/max over the last 4 hours, stores both with timestamp, purges aggregates older than 30 days
|
||||||
|
export async function aggregateLatency(now = Date.now()) {
|
||||||
|
const redis = getRedisClient();
|
||||||
|
const members = await redis.zrangebyscore(rawKey(), now - AGGREGATE_WINDOW_MS, '+inf');
|
||||||
|
const values = members
|
||||||
|
.map((m) => Number(m.slice(m.indexOf(':') + 1)))
|
||||||
|
.filter(Number.isFinite);
|
||||||
|
|
||||||
|
if (values.length > 0) {
|
||||||
|
const avg = Math.round((values.reduce((a, b) => a + b, 0) / values.length) * 100) / 100;
|
||||||
|
const max = Math.max(...values);
|
||||||
|
await redis
|
||||||
|
.multi()
|
||||||
|
.zadd(avgKey(), now, `${now}:${avg}`)
|
||||||
|
.zadd(maxKey(), now, `${now}:${max}`)
|
||||||
|
.exec();
|
||||||
|
healthLog.debug(`Aggregated ${values.length} samples: avg=${avg}ms max=${max}ms`);
|
||||||
|
} else {
|
||||||
|
healthLog.warn('No latency samples in the last 4h; skipping aggregate.');
|
||||||
|
}
|
||||||
|
|
||||||
|
const cutoff = now - AGGREGATE_RETENTION_MS;
|
||||||
|
await redis
|
||||||
|
.multi()
|
||||||
|
.zremrangebyscore(avgKey(), '-inf', cutoff)
|
||||||
|
.zremrangebyscore(maxKey(), '-inf', cutoff)
|
||||||
|
.exec();
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: runs one selftest cycle and records the latency in Redis
|
||||||
async function executeHealthCheckCycle() {
|
async function executeHealthCheckCycle() {
|
||||||
try {
|
try {
|
||||||
const result = await performHealthSelftest();
|
const result = await performHealthSelftest();
|
||||||
if (result.success) {
|
if (result.success) {
|
||||||
try {
|
try {
|
||||||
const redis = getRedisClient();
|
await recordLatency(result.durationMs);
|
||||||
await redis.set(env.redis.key, String(result.durationMs));
|
|
||||||
healthLog.debug(
|
healthLog.debug(
|
||||||
`Health selftest passed in ${result.durationMs}ms; recorded in redis key "${env.redis.key}"`
|
`Health selftest passed in ${result.durationMs}ms; recorded in redis key "${rawKey()}"`
|
||||||
);
|
);
|
||||||
} catch (redisErr) {
|
} catch (redisErr) {
|
||||||
healthLog.warn(`Failed to write latency to Redis: ${redisErr.message}`);
|
healthLog.warn(`Failed to write latency to Redis: ${redisErr.message}`);
|
||||||
@@ -139,6 +204,13 @@ export function startHealthMonitorService() {
|
|||||||
});
|
});
|
||||||
}, intervalMs);
|
}, intervalMs);
|
||||||
|
|
||||||
|
// Aggregate avg/max and purge old aggregates every 30 minutes
|
||||||
|
aggregateInterval = setInterval(() => {
|
||||||
|
aggregateLatency().catch((err) => {
|
||||||
|
healthLog.warn(`Latency aggregation failed: ${err.message}`);
|
||||||
|
});
|
||||||
|
}, AGGREGATE_INTERVAL_MS);
|
||||||
|
|
||||||
healthLog.info(`Health monitor service started (selftest every ${Math.round(intervalMs / 1000)}s via TCP).`);
|
healthLog.info(`Health monitor service started (selftest every ${Math.round(intervalMs / 1000)}s via TCP).`);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -153,6 +225,10 @@ export async function stopHealthMonitorService() {
|
|||||||
clearInterval(scheduledInterval);
|
clearInterval(scheduledInterval);
|
||||||
scheduledInterval = null;
|
scheduledInterval = null;
|
||||||
}
|
}
|
||||||
|
if (aggregateInterval) {
|
||||||
|
clearInterval(aggregateInterval);
|
||||||
|
aggregateInterval = null;
|
||||||
|
}
|
||||||
|
|
||||||
if (redisClient) {
|
if (redisClient) {
|
||||||
try {
|
try {
|
||||||
|
|||||||
Reference in New Issue
Block a user