258 lines
8.6 KiB
JavaScript
258 lines
8.6 KiB
JavaScript
// #region Imports & Configuration
|
|
import http from 'node:http';
|
|
import Redis from 'ioredis';
|
|
import env from '../config/env.js';
|
|
import { createLogger } from '../util/logger.js';
|
|
|
|
const healthLog = createLogger('health-monitor');
|
|
|
|
let scheduledTimer = null;
|
|
let scheduledInterval = null;
|
|
let aggregateInterval = null;
|
|
let isServiceRunning = false;
|
|
let redisClient = null;
|
|
|
|
const RAW_RETENTION_MS = 3 * 60 * 60 * 1000;
|
|
const AGGREGATE_WINDOW_MS = 4 * 60 * 60 * 1000;
|
|
const AGGREGATE_INTERVAL_MS = 30 * 60 * 1000;
|
|
const AGGREGATE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
|
|
|
|
// MARK: redis sorted-set keys (score = unix ms timestamp, member = "<ts>:<value>")
|
|
export const rawKey = () => `${env.redis.key}:raw`;
|
|
export const avgKey = () => `${env.redis.key}:avg`;
|
|
export const maxKey = () => `${env.redis.key}:max`;
|
|
// #endregion
|
|
|
|
// #region Redis Client Management
|
|
// MARK: resolves or creates the lazy Redis client
|
|
export function getRedisClient() {
|
|
if (!redisClient) {
|
|
const opts = env.redis.url
|
|
? env.redis.url
|
|
: {
|
|
host: env.redis.host,
|
|
port: env.redis.port,
|
|
password: env.redis.password || undefined,
|
|
db: env.redis.db || 0,
|
|
lazyConnect: true,
|
|
enableReadyCheck: false,
|
|
maxRetriesPerRequest: 1,
|
|
retryStrategy(times) {
|
|
return Math.min(times * 1000, 30000);
|
|
},
|
|
};
|
|
|
|
redisClient = new Redis(opts);
|
|
redisClient.on('error', (err) => {
|
|
healthLog.warn(`Redis connection error: ${err.message}`);
|
|
});
|
|
}
|
|
return redisClient;
|
|
}
|
|
// #endregion
|
|
|
|
// #region TCP Health API Selftest
|
|
// MARK: executes an HTTP GET /health call over TCP loopback and measures response time
|
|
export function performHealthSelftest() {
|
|
return new Promise((resolve, reject) => {
|
|
const start = performance.now();
|
|
const req = http.request(
|
|
{
|
|
hostname: '127.0.0.1',
|
|
port: env.port,
|
|
path: '/health',
|
|
method: 'GET',
|
|
headers: {
|
|
'User-Agent': 'Dashback-HealthMonitor/1.0',
|
|
'Accept': 'application/json',
|
|
},
|
|
timeout: 10000,
|
|
},
|
|
(res) => {
|
|
let data = '';
|
|
res.on('data', (chunk) => {
|
|
data += chunk;
|
|
});
|
|
res.on('end', () => {
|
|
const durationMs = Math.round((performance.now() - start) * 100) / 100;
|
|
if (res.statusCode >= 200 && res.statusCode < 300) {
|
|
resolve({ success: true, durationMs, statusCode: res.statusCode, body: data });
|
|
} else {
|
|
resolve({ success: false, durationMs, statusCode: res.statusCode, body: data });
|
|
}
|
|
});
|
|
}
|
|
);
|
|
|
|
req.on('error', (err) => {
|
|
const durationMs = Math.round((performance.now() - start) * 100) / 100;
|
|
reject({ error: err, durationMs });
|
|
});
|
|
|
|
req.on('timeout', () => {
|
|
req.destroy(new Error('TCP connection or request timed out'));
|
|
});
|
|
|
|
req.end();
|
|
});
|
|
}
|
|
|
|
// MARK: returns raw (3h), avg and max (30d) latency series as [{ timestamp, value }] oldest first
|
|
export async function getLatencySeries() {
|
|
const redis = getRedisClient();
|
|
const parse = (members) =>
|
|
members.map((m) => {
|
|
const i = m.indexOf(':');
|
|
return { timestamp: Number(m.slice(0, i)), value: Number(m.slice(i + 1)) };
|
|
});
|
|
const [raw, avg, max] = await Promise.all([
|
|
redis.zrange(rawKey(), 0, -1),
|
|
redis.zrange(avgKey(), 0, -1),
|
|
redis.zrange(maxKey(), 0, -1),
|
|
]);
|
|
return { raw: parse(raw), avg: parse(avg), max: parse(max) };
|
|
}
|
|
|
|
// MARK: stores one latency sample and trims samples older than 3 hours (round-robin window)
|
|
export async function recordLatency(durationMs, now = Date.now()) {
|
|
const redis = getRedisClient();
|
|
await redis
|
|
.multi()
|
|
.zadd(rawKey(), now, `${now}:${durationMs}`)
|
|
.zremrangebyscore(rawKey(), '-inf', now - RAW_RETENTION_MS)
|
|
.exec();
|
|
}
|
|
|
|
// MARK: computes avg/max over the last 4 hours, stores both with timestamp, purges aggregates older than 30 days
|
|
export async function aggregateLatency(now = Date.now()) {
|
|
const redis = getRedisClient();
|
|
const members = await redis.zrangebyscore(rawKey(), now - AGGREGATE_WINDOW_MS, '+inf');
|
|
const values = members
|
|
.map((m) => Number(m.slice(m.indexOf(':') + 1)))
|
|
.filter(Number.isFinite);
|
|
|
|
if (values.length > 0) {
|
|
const avg = Math.round((values.reduce((a, b) => a + b, 0) / values.length) * 100) / 100;
|
|
const max = Math.max(...values);
|
|
await redis
|
|
.multi()
|
|
.zadd(avgKey(), now, `${now}:${avg}`)
|
|
.zadd(maxKey(), now, `${now}:${max}`)
|
|
.exec();
|
|
healthLog.debug(`Aggregated ${values.length} samples: avg=${avg}ms max=${max}ms`);
|
|
} else {
|
|
healthLog.warn('No latency samples in the last 4h; skipping aggregate.');
|
|
}
|
|
|
|
const cutoff = now - AGGREGATE_RETENTION_MS;
|
|
await redis
|
|
.multi()
|
|
.zremrangebyscore(avgKey(), '-inf', cutoff)
|
|
.zremrangebyscore(maxKey(), '-inf', cutoff)
|
|
.exec();
|
|
}
|
|
|
|
// MARK: runs one selftest cycle and records the latency in Redis
|
|
async function executeHealthCheckCycle() {
|
|
try {
|
|
const result = await performHealthSelftest();
|
|
if (result.success) {
|
|
try {
|
|
await recordLatency(result.durationMs);
|
|
healthLog.debug(
|
|
`Health selftest passed in ${result.durationMs}ms; recorded in redis key "${rawKey()}"`
|
|
);
|
|
} catch (redisErr) {
|
|
healthLog.warn(`Failed to write latency to Redis: ${redisErr.message}`);
|
|
}
|
|
} else {
|
|
healthLog.warn(
|
|
`Health selftest returned HTTP ${result.statusCode} in ${result.durationMs}ms`
|
|
);
|
|
}
|
|
} catch (err) {
|
|
const msg = err?.error?.message || err?.message || String(err);
|
|
healthLog.warn(`Health selftest failed: ${msg}`);
|
|
}
|
|
}
|
|
// #endregion
|
|
|
|
// #region Service Lifecycle
|
|
// MARK: starts the periodic health check selftest service
|
|
export function startHealthMonitorService() {
|
|
if (isServiceRunning) return;
|
|
if (env.healthMonitor?.enabled === false) {
|
|
healthLog.info('Health monitor service is disabled by configuration.');
|
|
return;
|
|
}
|
|
|
|
isServiceRunning = true;
|
|
const intervalMs = env.healthMonitor?.intervalMs || 60_000;
|
|
|
|
// Run initial selftest after 1s warmup
|
|
scheduledTimer = setTimeout(() => {
|
|
scheduledTimer = null;
|
|
executeHealthCheckCycle().catch((err) => {
|
|
healthLog.error(`Initial health selftest error: ${err.message}`);
|
|
});
|
|
}, 1000);
|
|
|
|
// Schedule recurring selftest every minute
|
|
scheduledInterval = setInterval(() => {
|
|
executeHealthCheckCycle().catch((err) => {
|
|
healthLog.error(`Recurring health selftest error: ${err.message}`);
|
|
});
|
|
}, intervalMs);
|
|
|
|
// Aggregate avg/max and purge old aggregates every 30 minutes
|
|
aggregateInterval = setInterval(() => {
|
|
aggregateLatency().catch((err) => {
|
|
healthLog.warn(`Latency aggregation failed: ${err.message}`);
|
|
});
|
|
}, AGGREGATE_INTERVAL_MS);
|
|
|
|
healthLog.info(`Health monitor service started (selftest every ${Math.round(intervalMs / 1000)}s via TCP).`);
|
|
}
|
|
|
|
// MARK: stops the health check service and closes Redis client
|
|
export async function stopHealthMonitorService() {
|
|
isServiceRunning = false;
|
|
if (scheduledTimer) {
|
|
clearTimeout(scheduledTimer);
|
|
scheduledTimer = null;
|
|
}
|
|
if (scheduledInterval) {
|
|
clearInterval(scheduledInterval);
|
|
scheduledInterval = null;
|
|
}
|
|
if (aggregateInterval) {
|
|
clearInterval(aggregateInterval);
|
|
aggregateInterval = null;
|
|
}
|
|
|
|
if (redisClient) {
|
|
try {
|
|
await redisClient.quit();
|
|
} catch {
|
|
redisClient.disconnect();
|
|
} finally {
|
|
redisClient = null;
|
|
}
|
|
}
|
|
|
|
healthLog.info('Health monitor service stopped.');
|
|
}
|
|
|
|
// MARK: returns health monitor status
|
|
export function getHealthMonitorStatus() {
|
|
return {
|
|
enabled: env.healthMonitor?.enabled ?? true,
|
|
isRunning: isServiceRunning,
|
|
intervalMs: env.healthMonitor?.intervalMs || 60_000,
|
|
redisKey: env.redis.key,
|
|
redisHost: env.redis.host,
|
|
redisPort: env.redis.port,
|
|
};
|
|
}
|
|
// #endregion
|