u
This commit is contained in:
@@ -19,6 +19,7 @@ import { createLogger } from '../util/logger.js';
|
||||
import v1Router from '../v1/index.js';
|
||||
import { startBackendMonitor, stopBackendMonitor, isDevMode } from '../backendmonitor/server.js';
|
||||
import { startBackupService, stopBackupService } from '../services/backupService.js';
|
||||
import { startHealthMonitorService, stopHealthMonitorService } from '../services/healthMonitorService.js';
|
||||
|
||||
const apiLog = createLogger('api');
|
||||
|
||||
@@ -252,6 +253,12 @@ export async function startApi() {
|
||||
apiLog.error('Failed to start Backup Service:', err.message);
|
||||
}
|
||||
|
||||
try {
|
||||
startHealthMonitorService();
|
||||
} catch (err) {
|
||||
apiLog.error('Failed to start Health Monitor Service:', err.message);
|
||||
}
|
||||
|
||||
return server;
|
||||
})();
|
||||
|
||||
@@ -275,6 +282,7 @@ export async function stopApi() {
|
||||
// Same for the pooled sandbox threads that run `javascript_execute` / applets.
|
||||
await shutdownJavascriptSandboxWorkers();
|
||||
await stopBackupService();
|
||||
await stopHealthMonitorService();
|
||||
const closers = [];
|
||||
if (server) {
|
||||
const s = server;
|
||||
|
||||
@@ -47,7 +47,7 @@ function respondWithAuthPayload(res, req, status, body) {
|
||||
}
|
||||
|
||||
export const authMiddleware = async (req, res, next) => {
|
||||
if (req.path === '/' || req.path === '/api/auth/login' || req.path === '/api/auth/logout' || req.path === '/api/health' || req.path === '/api/openapi.json') {
|
||||
if (req.path === '/' || req.path === '/api/auth/login' || req.path === '/api/auth/logout' || req.path === '/api/health' || req.path === '/health' || req.path === '/api/openapi.json') {
|
||||
return next();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { ensureConnection } from '../../db/store.js';
|
||||
import { getStore } from '../../db/store.js';
|
||||
|
||||
/**
|
||||
* Registers a database connectivity health check route.
|
||||
@@ -12,12 +12,12 @@ export function register(app, basePath = '') {
|
||||
* get:
|
||||
* tags: [Health]
|
||||
* summary: Database health check
|
||||
* description: Verifies the database client can establish a connection.
|
||||
* description: Verifies database connectivity and transaction execution by updating a healthcheck sequence.
|
||||
* security: []
|
||||
* x-markdown: health.md
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Database reachable
|
||||
* description: Database reachable and healthy
|
||||
* content:
|
||||
* application/json:
|
||||
* schema:
|
||||
@@ -41,14 +41,20 @@ export function register(app, basePath = '') {
|
||||
*/
|
||||
// #region GET /health
|
||||
|
||||
app.get(`${basePath}/health`, async (_request, response) => {
|
||||
const healthHandler = async (_request, response) => {
|
||||
try {
|
||||
await ensureConnection();
|
||||
const store = await getStore();
|
||||
await store.healthcheck.check();
|
||||
response.status(200).json({ ok: true });
|
||||
} catch (error) {
|
||||
response.status(503).json({ ok: false, error: error.message });
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
app.get(`${basePath}/health`, healthHandler);
|
||||
if (basePath) {
|
||||
app.get('/health', healthHandler);
|
||||
}
|
||||
|
||||
// #endregion
|
||||
}
|
||||
|
||||
@@ -52,4 +52,16 @@ export default {
|
||||
hourlyRetentionCount: Number(process.env.BACKUP_HOURLY_RETENTION) || 50,
|
||||
dailyRetentionCount: Number(process.env.BACKUP_DAILY_RETENTION) || 50,
|
||||
},
|
||||
redis: {
|
||||
host: process.env.REDIS_HOST || '127.0.0.1',
|
||||
port: Number(process.env.REDIS_PORT) || 6379,
|
||||
password: process.env.REDIS_PASSWORD || undefined,
|
||||
db: Number(process.env.REDIS_DB) || 0,
|
||||
url: process.env.REDIS_URL || undefined,
|
||||
key: process.env.REDIS_KEY || process.env.REDIS_HEALTH_KEY || 'healthcheck:latency_ms',
|
||||
},
|
||||
healthMonitor: {
|
||||
enabled: process.env.HEALTH_MONITOR_ENABLED !== 'false',
|
||||
intervalMs: Number(process.env.HEALTH_MONITOR_INTERVAL_MS) || 60_000,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -55,6 +55,7 @@ const STORE_NAMESPACES = [
|
||||
'articlelist',
|
||||
'optionslist',
|
||||
'mcp',
|
||||
'healthcheck',
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
181
src/services/healthMonitorService.js
Normal file
181
src/services/healthMonitorService.js
Normal file
@@ -0,0 +1,181 @@
|
||||
// #region Imports & Configuration
|
||||
import http from 'node:http';
|
||||
import Redis from 'ioredis';
|
||||
import env from '../config/env.js';
|
||||
import { createLogger } from '../util/logger.js';
|
||||
|
||||
const healthLog = createLogger('health-monitor');
|
||||
|
||||
let scheduledTimer = null;
|
||||
let scheduledInterval = null;
|
||||
let isServiceRunning = false;
|
||||
let redisClient = null;
|
||||
// #endregion
|
||||
|
||||
// #region Redis Client Management
|
||||
// MARK: resolves or creates the lazy Redis client
|
||||
export function getRedisClient() {
|
||||
if (!redisClient) {
|
||||
const opts = env.redis.url
|
||||
? env.redis.url
|
||||
: {
|
||||
host: env.redis.host,
|
||||
port: env.redis.port,
|
||||
password: env.redis.password || undefined,
|
||||
db: env.redis.db || 0,
|
||||
lazyConnect: true,
|
||||
enableReadyCheck: false,
|
||||
maxRetriesPerRequest: 1,
|
||||
retryStrategy(times) {
|
||||
return Math.min(times * 1000, 30000);
|
||||
},
|
||||
};
|
||||
|
||||
redisClient = new Redis(opts);
|
||||
redisClient.on('error', (err) => {
|
||||
healthLog.warn(`Redis connection error: ${err.message}`);
|
||||
});
|
||||
}
|
||||
return redisClient;
|
||||
}
|
||||
// #endregion
|
||||
|
||||
// #region TCP Health API Selftest
|
||||
// MARK: executes an HTTP GET /health call over TCP loopback and measures response time
|
||||
export function performHealthSelftest() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const start = performance.now();
|
||||
const req = http.request(
|
||||
{
|
||||
hostname: '127.0.0.1',
|
||||
port: env.port,
|
||||
path: '/health',
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'User-Agent': 'Dashback-HealthMonitor/1.0',
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
timeout: 10000,
|
||||
},
|
||||
(res) => {
|
||||
let data = '';
|
||||
res.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
res.on('end', () => {
|
||||
const durationMs = Math.round((performance.now() - start) * 100) / 100;
|
||||
if (res.statusCode >= 200 && res.statusCode < 300) {
|
||||
resolve({ success: true, durationMs, statusCode: res.statusCode, body: data });
|
||||
} else {
|
||||
resolve({ success: false, durationMs, statusCode: res.statusCode, body: data });
|
||||
}
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
req.on('error', (err) => {
|
||||
const durationMs = Math.round((performance.now() - start) * 100) / 100;
|
||||
reject({ error: err, durationMs });
|
||||
});
|
||||
|
||||
req.on('timeout', () => {
|
||||
req.destroy(new Error('TCP connection or request timed out'));
|
||||
});
|
||||
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
// MARK: runs one selftest cycle and writes duration to Redis
|
||||
async function executeHealthCheckCycle() {
|
||||
try {
|
||||
const result = await performHealthSelftest();
|
||||
if (result.success) {
|
||||
try {
|
||||
const redis = getRedisClient();
|
||||
await redis.set(env.redis.key, String(result.durationMs));
|
||||
healthLog.debug(
|
||||
`Health selftest passed in ${result.durationMs}ms; recorded in redis key "${env.redis.key}"`
|
||||
);
|
||||
} catch (redisErr) {
|
||||
healthLog.warn(`Failed to write latency to Redis: ${redisErr.message}`);
|
||||
}
|
||||
} else {
|
||||
healthLog.warn(
|
||||
`Health selftest returned HTTP ${result.statusCode} in ${result.durationMs}ms`
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err?.error?.message || err?.message || String(err);
|
||||
healthLog.warn(`Health selftest failed: ${msg}`);
|
||||
}
|
||||
}
|
||||
// #endregion
|
||||
|
||||
// #region Service Lifecycle
|
||||
// MARK: starts the periodic health check selftest service
|
||||
export function startHealthMonitorService() {
|
||||
if (isServiceRunning) return;
|
||||
if (env.healthMonitor?.enabled === false) {
|
||||
healthLog.info('Health monitor service is disabled by configuration.');
|
||||
return;
|
||||
}
|
||||
|
||||
isServiceRunning = true;
|
||||
const intervalMs = env.healthMonitor?.intervalMs || 60_000;
|
||||
|
||||
// Run initial selftest after 1s warmup
|
||||
scheduledTimer = setTimeout(() => {
|
||||
scheduledTimer = null;
|
||||
executeHealthCheckCycle().catch((err) => {
|
||||
healthLog.error(`Initial health selftest error: ${err.message}`);
|
||||
});
|
||||
}, 1000);
|
||||
|
||||
// Schedule recurring selftest every minute
|
||||
scheduledInterval = setInterval(() => {
|
||||
executeHealthCheckCycle().catch((err) => {
|
||||
healthLog.error(`Recurring health selftest error: ${err.message}`);
|
||||
});
|
||||
}, intervalMs);
|
||||
|
||||
healthLog.info(`Health monitor service started (selftest every ${Math.round(intervalMs / 1000)}s via TCP).`);
|
||||
}
|
||||
|
||||
// MARK: stops the health check service and closes Redis client
|
||||
export async function stopHealthMonitorService() {
|
||||
isServiceRunning = false;
|
||||
if (scheduledTimer) {
|
||||
clearTimeout(scheduledTimer);
|
||||
scheduledTimer = null;
|
||||
}
|
||||
if (scheduledInterval) {
|
||||
clearInterval(scheduledInterval);
|
||||
scheduledInterval = null;
|
||||
}
|
||||
|
||||
if (redisClient) {
|
||||
try {
|
||||
await redisClient.quit();
|
||||
} catch {
|
||||
redisClient.disconnect();
|
||||
} finally {
|
||||
redisClient = null;
|
||||
}
|
||||
}
|
||||
|
||||
healthLog.info('Health monitor service stopped.');
|
||||
}
|
||||
|
||||
// MARK: returns health monitor status
|
||||
export function getHealthMonitorStatus() {
|
||||
return {
|
||||
enabled: env.healthMonitor?.enabled ?? true,
|
||||
isRunning: isServiceRunning,
|
||||
intervalMs: env.healthMonitor?.intervalMs || 60_000,
|
||||
redisKey: env.redis.key,
|
||||
redisHost: env.redis.host,
|
||||
redisPort: env.redis.port,
|
||||
};
|
||||
}
|
||||
// #endregion
|
||||
Reference in New Issue
Block a user