u
This commit is contained in:
@@ -159,6 +159,7 @@ Generated from `@openapi` annotations in [`src/api/routes/`](../../src/api/route
|
||||
| [`routes/entity/update.md`](routes/entity/update.md) | `PATCH /entity/{id}` |
|
||||
| [`routes/events.md`](routes/events.md) | `GET /events` |
|
||||
| [`routes/health.md`](routes/health.md) | `GET /health` |
|
||||
| [`routes/health/latency.md`](routes/health/latency.md) | `GET /health/latency` |
|
||||
| [`routes/image/get.md`](routes/image/get.md) | `GET /image/{fullHash}` |
|
||||
| [`routes/labeltemplate/list.md`](routes/labeltemplate/list.md) | `GET /labeltemplate` |
|
||||
| [`routes/labeltemplate/create.md`](routes/labeltemplate/create.md) | `POST /labeltemplate` |
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Dash API Routes Overview (All)
|
||||
|
||||
Overview of all 263 API endpoints across all modules.
|
||||
Overview of all 264 API endpoints across all modules.
|
||||
|
||||
### Notation
|
||||
|
||||
@@ -498,6 +498,9 @@ _none_
|
||||
#### Database health check — `GET /health`
|
||||
_none_
|
||||
- Returns: `200 { ok: bool }`
|
||||
#### Health latency history — `GET /health/latency`
|
||||
_none_
|
||||
- Returns: `200 { raw: { timestamp: int, value: num }[], avg: { timestamp: int, value: num }[], max: { timestamp: int, value: num }[] }`
|
||||
|
||||
### Section: /image
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Dash API Routes Overview
|
||||
|
||||
Overview of all 186 API endpoints across all modules.
|
||||
Overview of all 187 API endpoints across all modules.
|
||||
|
||||
### Notation
|
||||
|
||||
@@ -381,6 +381,12 @@ _none_
|
||||
- Params: `id: int, body: { name?: str, firstName?: str|null, number?: str|null }`
|
||||
- Returns: `200 { entity: T19 }`
|
||||
|
||||
### Section: /health
|
||||
|
||||
#### Health latency history — `GET /health/latency`
|
||||
_none_
|
||||
- Returns: `200 { raw: { timestamp: int, value: num }[], avg: { timestamp: int, value: num }[], max: { timestamp: int, value: num }[] }`
|
||||
|
||||
### Section: /labeltemplate
|
||||
|
||||
#### List label templates — `GET /labeltemplate`
|
||||
|
||||
45
docs/api/routes/health/latency.md
Normal file
45
docs/api/routes/health/latency.md
Normal file
@@ -0,0 +1,45 @@
|
||||
# `GET /health/latency`
|
||||
|
||||
Protected. Requires a valid session cookie (`credentials: 'include'`).
|
||||
|
||||
Health latency history
|
||||
|
||||
Returns the database healthcheck latency (ms) as three series, each oldest first with unix-ms timestamps. `raw` holds the per-minute samples of the last 3 hours; `avg` and `max` hold the 30-minute aggregates (over the last 4 hours of samples) kept for 30 days. Requires a valid JWT; no company selection needed.
|
||||
|
||||
## Response `200`
|
||||
|
||||
Latency series
|
||||
|
||||
```json
|
||||
{
|
||||
"raw": [
|
||||
{
|
||||
"timestamp": 1,
|
||||
"value": 1
|
||||
}
|
||||
],
|
||||
"avg": [
|
||||
{
|
||||
"timestamp": 1,
|
||||
"value": 1
|
||||
}
|
||||
],
|
||||
"max": [
|
||||
{
|
||||
"timestamp": 1,
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Errors
|
||||
|
||||
| Status | Description |
|
||||
|--------|-------------|
|
||||
| 401 | Unauthorized |
|
||||
| 500 | Redis read failed |
|
||||
|
||||
---
|
||||
|
||||
_Generated from `@openapi` JSDoc. Do not edit by hand — run `npm run docs:api`._
|
||||
@@ -1,4 +1,5 @@
|
||||
import { getStore } from '../../db/store.js';
|
||||
import { getLatencySeries } from '../../services/healthMonitorService.js';
|
||||
|
||||
/**
|
||||
* Registers a database connectivity health check route.
|
||||
@@ -57,4 +58,57 @@ export function register(app, basePath = '') {
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
/**
|
||||
* @openapi
|
||||
* /health/latency:
|
||||
* get:
|
||||
* tags: [Health]
|
||||
* summary: Health latency history
|
||||
* description: Returns the database healthcheck latency (ms) as three series, each oldest first with unix-ms timestamps. `raw` holds the per-minute samples of the last 3 hours; `avg` and `max` hold the 30-minute aggregates (over the last 4 hours of samples) kept for 30 days. Requires a valid JWT; no company selection needed.
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Latency series
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
* type: object
|
||||
* properties:
|
||||
* raw:
|
||||
* type: array
|
||||
* items: { $ref: '#/components/schemas/LatencyPoint' }
|
||||
* avg:
|
||||
* type: array
|
||||
* items: { $ref: '#/components/schemas/LatencyPoint' }
|
||||
* max:
|
||||
* type: array
|
||||
* items: { $ref: '#/components/schemas/LatencyPoint' }
|
||||
* 401:
|
||||
* description: Unauthorized
|
||||
* 500:
|
||||
* description: Redis read failed
|
||||
* components:
|
||||
* schemas:
|
||||
* LatencyPoint:
|
||||
* type: object
|
||||
* properties:
|
||||
* timestamp:
|
||||
* type: integer
|
||||
* description: Unix epoch milliseconds
|
||||
* value:
|
||||
* type: number
|
||||
* description: Latency in milliseconds
|
||||
*/
|
||||
// #region GET /health/latency
|
||||
|
||||
app.get(`${basePath}/health/latency`, async (_request, response) => {
|
||||
try {
|
||||
response.json(await getLatencySeries());
|
||||
} catch (error) {
|
||||
console.error('GET /health/latency failed:', error);
|
||||
response.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// #endregion
|
||||
}
|
||||
|
||||
@@ -8,8 +8,19 @@ const healthLog = createLogger('health-monitor');
|
||||
|
||||
let scheduledTimer = null;
|
||||
let scheduledInterval = null;
|
||||
let aggregateInterval = null;
|
||||
let isServiceRunning = false;
|
||||
let redisClient = null;
|
||||
|
||||
const RAW_RETENTION_MS = 3 * 60 * 60 * 1000;
|
||||
const AGGREGATE_WINDOW_MS = 4 * 60 * 60 * 1000;
|
||||
const AGGREGATE_INTERVAL_MS = 30 * 60 * 1000;
|
||||
const AGGREGATE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
|
||||
|
||||
// MARK: redis sorted-set keys (score = unix ms timestamp, member = "<ts>:<value>")
|
||||
export const rawKey = () => `${env.redis.key}:raw`;
|
||||
export const avgKey = () => `${env.redis.key}:avg`;
|
||||
export const maxKey = () => `${env.redis.key}:max`;
|
||||
// #endregion
|
||||
|
||||
// #region Redis Client Management
|
||||
@@ -86,16 +97,70 @@ export function performHealthSelftest() {
|
||||
});
|
||||
}
|
||||
|
||||
// MARK: runs one selftest cycle and writes duration to Redis
|
||||
// MARK: returns raw (3h), avg and max (30d) latency series as [{ timestamp, value }] oldest first
|
||||
export async function getLatencySeries() {
|
||||
const redis = getRedisClient();
|
||||
const parse = (members) =>
|
||||
members.map((m) => {
|
||||
const i = m.indexOf(':');
|
||||
return { timestamp: Number(m.slice(0, i)), value: Number(m.slice(i + 1)) };
|
||||
});
|
||||
const [raw, avg, max] = await Promise.all([
|
||||
redis.zrange(rawKey(), 0, -1),
|
||||
redis.zrange(avgKey(), 0, -1),
|
||||
redis.zrange(maxKey(), 0, -1),
|
||||
]);
|
||||
return { raw: parse(raw), avg: parse(avg), max: parse(max) };
|
||||
}
|
||||
|
||||
// MARK: stores one latency sample and trims samples older than 3 hours (round-robin window)
|
||||
export async function recordLatency(durationMs, now = Date.now()) {
|
||||
const redis = getRedisClient();
|
||||
await redis
|
||||
.multi()
|
||||
.zadd(rawKey(), now, `${now}:${durationMs}`)
|
||||
.zremrangebyscore(rawKey(), '-inf', now - RAW_RETENTION_MS)
|
||||
.exec();
|
||||
}
|
||||
|
||||
// MARK: computes avg/max over the last 4 hours, stores both with timestamp, purges aggregates older than 30 days
|
||||
export async function aggregateLatency(now = Date.now()) {
|
||||
const redis = getRedisClient();
|
||||
const members = await redis.zrangebyscore(rawKey(), now - AGGREGATE_WINDOW_MS, '+inf');
|
||||
const values = members
|
||||
.map((m) => Number(m.slice(m.indexOf(':') + 1)))
|
||||
.filter(Number.isFinite);
|
||||
|
||||
if (values.length > 0) {
|
||||
const avg = Math.round((values.reduce((a, b) => a + b, 0) / values.length) * 100) / 100;
|
||||
const max = Math.max(...values);
|
||||
await redis
|
||||
.multi()
|
||||
.zadd(avgKey(), now, `${now}:${avg}`)
|
||||
.zadd(maxKey(), now, `${now}:${max}`)
|
||||
.exec();
|
||||
healthLog.debug(`Aggregated ${values.length} samples: avg=${avg}ms max=${max}ms`);
|
||||
} else {
|
||||
healthLog.warn('No latency samples in the last 4h; skipping aggregate.');
|
||||
}
|
||||
|
||||
const cutoff = now - AGGREGATE_RETENTION_MS;
|
||||
await redis
|
||||
.multi()
|
||||
.zremrangebyscore(avgKey(), '-inf', cutoff)
|
||||
.zremrangebyscore(maxKey(), '-inf', cutoff)
|
||||
.exec();
|
||||
}
|
||||
|
||||
// MARK: runs one selftest cycle and records the latency in Redis
|
||||
async function executeHealthCheckCycle() {
|
||||
try {
|
||||
const result = await performHealthSelftest();
|
||||
if (result.success) {
|
||||
try {
|
||||
const redis = getRedisClient();
|
||||
await redis.set(env.redis.key, String(result.durationMs));
|
||||
await recordLatency(result.durationMs);
|
||||
healthLog.debug(
|
||||
`Health selftest passed in ${result.durationMs}ms; recorded in redis key "${env.redis.key}"`
|
||||
`Health selftest passed in ${result.durationMs}ms; recorded in redis key "${rawKey()}"`
|
||||
);
|
||||
} catch (redisErr) {
|
||||
healthLog.warn(`Failed to write latency to Redis: ${redisErr.message}`);
|
||||
@@ -139,6 +204,13 @@ export function startHealthMonitorService() {
|
||||
});
|
||||
}, intervalMs);
|
||||
|
||||
// Aggregate avg/max and purge old aggregates every 30 minutes
|
||||
aggregateInterval = setInterval(() => {
|
||||
aggregateLatency().catch((err) => {
|
||||
healthLog.warn(`Latency aggregation failed: ${err.message}`);
|
||||
});
|
||||
}, AGGREGATE_INTERVAL_MS);
|
||||
|
||||
healthLog.info(`Health monitor service started (selftest every ${Math.round(intervalMs / 1000)}s via TCP).`);
|
||||
}
|
||||
|
||||
@@ -153,6 +225,10 @@ export async function stopHealthMonitorService() {
|
||||
clearInterval(scheduledInterval);
|
||||
scheduledInterval = null;
|
||||
}
|
||||
if (aggregateInterval) {
|
||||
clearInterval(aggregateInterval);
|
||||
aggregateInterval = null;
|
||||
}
|
||||
|
||||
if (redisClient) {
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user