// #region Imports & Configuration import http from 'node:http'; import Redis from 'ioredis'; import env from '../config/env.js'; import { createLogger } from '../util/logger.js'; const healthLog = createLogger('health-monitor'); let scheduledTimer = null; let scheduledInterval = null; let aggregateInterval = null; let isServiceRunning = false; let redisClient = null; const RAW_RETENTION_MS = 3 * 60 * 60 * 1000; const AGGREGATE_WINDOW_MS = 4 * 60 * 60 * 1000; const AGGREGATE_INTERVAL_MS = 30 * 60 * 1000; const AGGREGATE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; // MARK: redis sorted-set keys (score = unix ms timestamp, member = ":") export const rawKey = () => `${env.redis.key}:raw`; export const avgKey = () => `${env.redis.key}:avg`; export const maxKey = () => `${env.redis.key}:max`; // #endregion // #region Redis Client Management // MARK: resolves or creates the lazy Redis client export function getRedisClient() { if (!redisClient) { const opts = env.redis.url ? env.redis.url : { host: env.redis.host, port: env.redis.port, password: env.redis.password || undefined, db: env.redis.db || 0, lazyConnect: true, enableReadyCheck: false, maxRetriesPerRequest: 1, retryStrategy(times) { return Math.min(times * 1000, 30000); }, }; redisClient = new Redis(opts); redisClient.on('error', (err) => { healthLog.warn(`Redis connection error: ${err.message}`); }); } return redisClient; } // #endregion // #region TCP Health API Selftest // MARK: executes an HTTP GET /health call over TCP loopback and measures response time export function performHealthSelftest() { return new Promise((resolve, reject) => { const start = performance.now(); const req = http.request( { hostname: '127.0.0.1', port: env.port, path: '/health', method: 'GET', headers: { 'User-Agent': 'Dashback-HealthMonitor/1.0', 'Accept': 'application/json', }, timeout: 10000, }, (res) => { let data = ''; res.on('data', (chunk) => { data += chunk; }); res.on('end', () => { const durationMs = Math.round((performance.now() - start) * 100) / 100; if (res.statusCode >= 200 && res.statusCode < 300) { resolve({ success: true, durationMs, statusCode: res.statusCode, body: data }); } else { resolve({ success: false, durationMs, statusCode: res.statusCode, body: data }); } }); } ); req.on('error', (err) => { const durationMs = Math.round((performance.now() - start) * 100) / 100; reject({ error: err, durationMs }); }); req.on('timeout', () => { req.destroy(new Error('TCP connection or request timed out')); }); req.end(); }); } // MARK: returns raw (3h), avg and max (30d) latency series as [{ timestamp, value }] oldest first export async function getLatencySeries() { const redis = getRedisClient(); const parse = (members) => members.map((m) => { const i = m.indexOf(':'); return { timestamp: Number(m.slice(0, i)), value: Number(m.slice(i + 1)) }; }); const [raw, avg, max] = await Promise.all([ redis.zrange(rawKey(), 0, -1), redis.zrange(avgKey(), 0, -1), redis.zrange(maxKey(), 0, -1), ]); return { raw: parse(raw), avg: parse(avg), max: parse(max) }; } // MARK: stores one latency sample and trims samples older than 3 hours (round-robin window) export async function recordLatency(durationMs, now = Date.now()) { const redis = getRedisClient(); await redis .multi() .zadd(rawKey(), now, `${now}:${durationMs}`) .zremrangebyscore(rawKey(), '-inf', now - RAW_RETENTION_MS) .exec(); } // MARK: computes avg/max over the last 4 hours, stores both with timestamp, purges aggregates older than 30 days export async function aggregateLatency(now = Date.now()) { const redis = getRedisClient(); const members = await redis.zrangebyscore(rawKey(), now - AGGREGATE_WINDOW_MS, '+inf'); const values = members .map((m) => Number(m.slice(m.indexOf(':') + 1))) .filter(Number.isFinite); if (values.length > 0) { const avg = Math.round((values.reduce((a, b) => a + b, 0) / values.length) * 100) / 100; const max = Math.max(...values); await redis .multi() .zadd(avgKey(), now, `${now}:${avg}`) .zadd(maxKey(), now, `${now}:${max}`) .exec(); healthLog.debug(`Aggregated ${values.length} samples: avg=${avg}ms max=${max}ms`); } else { healthLog.warn('No latency samples in the last 4h; skipping aggregate.'); } const cutoff = now - AGGREGATE_RETENTION_MS; await redis .multi() .zremrangebyscore(avgKey(), '-inf', cutoff) .zremrangebyscore(maxKey(), '-inf', cutoff) .exec(); } // MARK: runs one selftest cycle and records the latency in Redis async function executeHealthCheckCycle() { try { const result = await performHealthSelftest(); if (result.success) { try { await recordLatency(result.durationMs); healthLog.debug( `Health selftest passed in ${result.durationMs}ms; recorded in redis key "${rawKey()}"` ); } catch (redisErr) { healthLog.warn(`Failed to write latency to Redis: ${redisErr.message}`); } } else { healthLog.warn( `Health selftest returned HTTP ${result.statusCode} in ${result.durationMs}ms` ); } } catch (err) { const msg = err?.error?.message || err?.message || String(err); healthLog.warn(`Health selftest failed: ${msg}`); } } // #endregion // #region Service Lifecycle // MARK: starts the periodic health check selftest service export function startHealthMonitorService() { if (isServiceRunning) return; if (env.healthMonitor?.enabled === false) { healthLog.info('Health monitor service is disabled by configuration.'); return; } isServiceRunning = true; const intervalMs = env.healthMonitor?.intervalMs || 60_000; // Run initial selftest after 1s warmup scheduledTimer = setTimeout(() => { scheduledTimer = null; executeHealthCheckCycle().catch((err) => { healthLog.error(`Initial health selftest error: ${err.message}`); }); }, 1000); // Schedule recurring selftest every minute scheduledInterval = setInterval(() => { executeHealthCheckCycle().catch((err) => { healthLog.error(`Recurring health selftest error: ${err.message}`); }); }, intervalMs); // Aggregate avg/max and purge old aggregates every 30 minutes aggregateInterval = setInterval(() => { aggregateLatency().catch((err) => { healthLog.warn(`Latency aggregation failed: ${err.message}`); }); }, AGGREGATE_INTERVAL_MS); healthLog.info(`Health monitor service started (selftest every ${Math.round(intervalMs / 1000)}s via TCP).`); } // MARK: stops the health check service and closes Redis client export async function stopHealthMonitorService() { isServiceRunning = false; if (scheduledTimer) { clearTimeout(scheduledTimer); scheduledTimer = null; } if (scheduledInterval) { clearInterval(scheduledInterval); scheduledInterval = null; } if (aggregateInterval) { clearInterval(aggregateInterval); aggregateInterval = null; } if (redisClient) { try { await redisClient.quit(); } catch { redisClient.disconnect(); } finally { redisClient = null; } } healthLog.info('Health monitor service stopped.'); } // MARK: returns health monitor status export function getHealthMonitorStatus() { return { enabled: env.healthMonitor?.enabled ?? true, isRunning: isServiceRunning, intervalMs: env.healthMonitor?.intervalMs || 60_000, redisKey: env.redis.key, redisHost: env.redis.host, redisPort: env.redis.port, }; } // #endregion