From a717a05e8d2dec77b1fecc22700cbc8f448b634a Mon Sep 17 00:00:00 2001 From: seb Date: Sun, 18 Oct 2026 20:48:50 +0200 Subject: [PATCH] u --- .env.example | 13 ++ docs/api/routes/health.md | 4 +- package-lock.json | 65 +++++++++- package.json | 1 + src/api/index.js | 8 ++ src/api/middleware/auth.js | 2 +- src/api/routes/health.js | 18 ++- src/config/env.js | 12 ++ src/db/store.js | 1 + src/services/healthMonitorService.js | 181 +++++++++++++++++++++++++++ 10 files changed, 294 insertions(+), 11 deletions(-) create mode 100644 src/services/healthMonitorService.js diff --git a/.env.example b/.env.example index 2276f1b..b9194ab 100644 --- a/.env.example +++ b/.env.example @@ -51,4 +51,17 @@ OPENROUTER_PAID_MODEL=openai/gpt-6-luna:floor # BACKUP_HOURLY_RETENTION=50 # BACKUP_DAILY_RETENTION=50 +# Redis (Healthcheck Latency Metric) +REDIS_HOST=127.0.0.1 +REDIS_PORT=6379 +REDIS_PASSWORD= +REDIS_DB=0 +REDIS_KEY=healthcheck:latency_ms +# REDIS_URL=redis://127.0.0.1:6379 + +# Health Monitor Service +# HEALTH_MONITOR_ENABLED=true +# HEALTH_MONITOR_INTERVAL_MS=60000 + + diff --git a/docs/api/routes/health.md b/docs/api/routes/health.md index 206d369..fb05a31 100644 --- a/docs/api/routes/health.md +++ b/docs/api/routes/health.md @@ -4,11 +4,11 @@ Public. No cookie required. Database health check -Verifies the database client can establish a connection. +Verifies database connectivity and transaction execution by updating a healthcheck sequence. ## Response `200` -Database reachable +Database reachable and healthy ```json { diff --git a/package-lock.json b/package-lock.json index 0f5113a..ed61c71 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "google-auth-library": "^11.3.1", "ink": "^8.0.0", "ink-text-input": "^6.0.0", + "ioredis": "^6.0.0", "jsonwebtoken": "^9.0.3", "jszip": "^3.10.2", "libdashstore": "file:../dashdbschema", @@ -53,6 +54,7 @@ "dotenv": "^18.0.8", "pdfconvert-wasm": "^0.3.0", "sharp": "^0.35.5", + "skelehash-wasm": "^0.1.0", "thumbhash": "^0.1.1" }, "devDependencies": { @@ -70,11 +72,10 @@ } }, "../llmorbiter": { - "name": "llmorbiter", "version": "0.1.0", "license": "0BSD", "dependencies": { - "dotenv": "^18.0.5" + "dotenv": "^18.0.8" }, "engines": { "node": ">=18.0.0" @@ -769,6 +770,12 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@ioredis/commands": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-2.0.0.tgz", + "integrity": "sha512-vrx0AE/T0h7cRZwfo1M39Cr+ZhZrkf0V8mQN75wucKCxCLD9l/VX6no3gFvrLqD1IlG/1LtzWovqEw3t0Vr9zg==", + "license": "MIT" + }, "node_modules/@isaacs/cliui": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-9.0.0.tgz", @@ -1849,6 +1856,15 @@ "node": ">=0.8" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz", + "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/code-excerpt": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/code-excerpt/-/code-excerpt-4.0.0.tgz", @@ -2198,6 +2214,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -3460,6 +3485,27 @@ "node": ">= 0.4" } }, + "node_modules/ioredis": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-6.0.0.tgz", + "integrity": "sha512-f+Dtubxfpf6KYFq7WVXJoOLn0bk4TJrMrN9SzeE+jrWrCWj7XX3fA6vkryafhADX+GMymRxgDJDOI33COkJc0w==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "2.0.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ip-address": { "version": "10.7.4", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.4.tgz", @@ -5125,6 +5171,15 @@ "node": ">=10" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", @@ -5717,6 +5772,12 @@ "node": ">=10" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", diff --git a/package.json b/package.json index 4983d7a..9603c0d 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "google-auth-library": "^11.3.1", "ink": "^8.0.0", "ink-text-input": "^6.0.0", + "ioredis": "^6.0.0", "jsonwebtoken": "^9.0.3", "jszip": "^3.10.2", "libdashstore": "file:../dashdbschema", diff --git a/src/api/index.js b/src/api/index.js index 661255d..7831db9 100644 --- a/src/api/index.js +++ b/src/api/index.js @@ -19,6 +19,7 @@ import { createLogger } from '../util/logger.js'; import v1Router from '../v1/index.js'; import { startBackendMonitor, stopBackendMonitor, isDevMode } from '../backendmonitor/server.js'; import { startBackupService, stopBackupService } from '../services/backupService.js'; +import { startHealthMonitorService, stopHealthMonitorService } from '../services/healthMonitorService.js'; const apiLog = createLogger('api'); @@ -252,6 +253,12 @@ export async function startApi() { apiLog.error('Failed to start Backup Service:', err.message); } + try { + startHealthMonitorService(); + } catch (err) { + apiLog.error('Failed to start Health Monitor Service:', err.message); + } + return server; })(); @@ -275,6 +282,7 @@ export async function stopApi() { // Same for the pooled sandbox threads that run `javascript_execute` / applets. await shutdownJavascriptSandboxWorkers(); await stopBackupService(); + await stopHealthMonitorService(); const closers = []; if (server) { const s = server; diff --git a/src/api/middleware/auth.js b/src/api/middleware/auth.js index e3d948c..da60e5b 100644 --- a/src/api/middleware/auth.js +++ b/src/api/middleware/auth.js @@ -47,7 +47,7 @@ function respondWithAuthPayload(res, req, status, body) { } export const authMiddleware = async (req, res, next) => { - if (req.path === '/' || req.path === '/api/auth/login' || req.path === '/api/auth/logout' || req.path === '/api/health' || req.path === '/api/openapi.json') { + if (req.path === '/' || req.path === '/api/auth/login' || req.path === '/api/auth/logout' || req.path === '/api/health' || req.path === '/health' || req.path === '/api/openapi.json') { return next(); } diff --git a/src/api/routes/health.js b/src/api/routes/health.js index 784baf1..75ceb19 100644 --- a/src/api/routes/health.js +++ b/src/api/routes/health.js @@ -1,4 +1,4 @@ -import { ensureConnection } from '../../db/store.js'; +import { getStore } from '../../db/store.js'; /** * Registers a database connectivity health check route. @@ -12,12 +12,12 @@ export function register(app, basePath = '') { * get: * tags: [Health] * summary: Database health check - * description: Verifies the database client can establish a connection. + * description: Verifies database connectivity and transaction execution by updating a healthcheck sequence. * security: [] * x-markdown: health.md * responses: * 200: - * description: Database reachable + * description: Database reachable and healthy * content: * application/json: * schema: @@ -41,14 +41,20 @@ export function register(app, basePath = '') { */ // #region GET /health - app.get(`${basePath}/health`, async (_request, response) => { + const healthHandler = async (_request, response) => { try { - await ensureConnection(); + const store = await getStore(); + await store.healthcheck.check(); response.status(200).json({ ok: true }); } catch (error) { response.status(503).json({ ok: false, error: error.message }); } - }); + }; + + app.get(`${basePath}/health`, healthHandler); + if (basePath) { + app.get('/health', healthHandler); + } // #endregion } diff --git a/src/config/env.js b/src/config/env.js index efd0c22..4849dac 100644 --- a/src/config/env.js +++ b/src/config/env.js @@ -52,4 +52,16 @@ export default { hourlyRetentionCount: Number(process.env.BACKUP_HOURLY_RETENTION) || 50, dailyRetentionCount: Number(process.env.BACKUP_DAILY_RETENTION) || 50, }, + redis: { + host: process.env.REDIS_HOST || '127.0.0.1', + port: Number(process.env.REDIS_PORT) || 6379, + password: process.env.REDIS_PASSWORD || undefined, + db: Number(process.env.REDIS_DB) || 0, + url: process.env.REDIS_URL || undefined, + key: process.env.REDIS_KEY || process.env.REDIS_HEALTH_KEY || 'healthcheck:latency_ms', + }, + healthMonitor: { + enabled: process.env.HEALTH_MONITOR_ENABLED !== 'false', + intervalMs: Number(process.env.HEALTH_MONITOR_INTERVAL_MS) || 60_000, + }, }; diff --git a/src/db/store.js b/src/db/store.js index 5ab221f..3a0fc65 100644 --- a/src/db/store.js +++ b/src/db/store.js @@ -55,6 +55,7 @@ const STORE_NAMESPACES = [ 'articlelist', 'optionslist', 'mcp', + 'healthcheck', ]; /** diff --git a/src/services/healthMonitorService.js b/src/services/healthMonitorService.js new file mode 100644 index 0000000..351f959 --- /dev/null +++ b/src/services/healthMonitorService.js @@ -0,0 +1,181 @@ +// #region Imports & Configuration +import http from 'node:http'; +import Redis from 'ioredis'; +import env from '../config/env.js'; +import { createLogger } from '../util/logger.js'; + +const healthLog = createLogger('health-monitor'); + +let scheduledTimer = null; +let scheduledInterval = null; +let isServiceRunning = false; +let redisClient = null; +// #endregion + +// #region Redis Client Management +// MARK: resolves or creates the lazy Redis client +export function getRedisClient() { + if (!redisClient) { + const opts = env.redis.url + ? env.redis.url + : { + host: env.redis.host, + port: env.redis.port, + password: env.redis.password || undefined, + db: env.redis.db || 0, + lazyConnect: true, + enableReadyCheck: false, + maxRetriesPerRequest: 1, + retryStrategy(times) { + return Math.min(times * 1000, 30000); + }, + }; + + redisClient = new Redis(opts); + redisClient.on('error', (err) => { + healthLog.warn(`Redis connection error: ${err.message}`); + }); + } + return redisClient; +} +// #endregion + +// #region TCP Health API Selftest +// MARK: executes an HTTP GET /health call over TCP loopback and measures response time +export function performHealthSelftest() { + return new Promise((resolve, reject) => { + const start = performance.now(); + const req = http.request( + { + hostname: '127.0.0.1', + port: env.port, + path: '/health', + method: 'GET', + headers: { + 'User-Agent': 'Dashback-HealthMonitor/1.0', + 'Accept': 'application/json', + }, + timeout: 10000, + }, + (res) => { + let data = ''; + res.on('data', (chunk) => { + data += chunk; + }); + res.on('end', () => { + const durationMs = Math.round((performance.now() - start) * 100) / 100; + if (res.statusCode >= 200 && res.statusCode < 300) { + resolve({ success: true, durationMs, statusCode: res.statusCode, body: data }); + } else { + resolve({ success: false, durationMs, statusCode: res.statusCode, body: data }); + } + }); + } + ); + + req.on('error', (err) => { + const durationMs = Math.round((performance.now() - start) * 100) / 100; + reject({ error: err, durationMs }); + }); + + req.on('timeout', () => { + req.destroy(new Error('TCP connection or request timed out')); + }); + + req.end(); + }); +} + +// MARK: runs one selftest cycle and writes duration to Redis +async function executeHealthCheckCycle() { + try { + const result = await performHealthSelftest(); + if (result.success) { + try { + const redis = getRedisClient(); + await redis.set(env.redis.key, String(result.durationMs)); + healthLog.debug( + `Health selftest passed in ${result.durationMs}ms; recorded in redis key "${env.redis.key}"` + ); + } catch (redisErr) { + healthLog.warn(`Failed to write latency to Redis: ${redisErr.message}`); + } + } else { + healthLog.warn( + `Health selftest returned HTTP ${result.statusCode} in ${result.durationMs}ms` + ); + } + } catch (err) { + const msg = err?.error?.message || err?.message || String(err); + healthLog.warn(`Health selftest failed: ${msg}`); + } +} +// #endregion + +// #region Service Lifecycle +// MARK: starts the periodic health check selftest service +export function startHealthMonitorService() { + if (isServiceRunning) return; + if (env.healthMonitor?.enabled === false) { + healthLog.info('Health monitor service is disabled by configuration.'); + return; + } + + isServiceRunning = true; + const intervalMs = env.healthMonitor?.intervalMs || 60_000; + + // Run initial selftest after 1s warmup + scheduledTimer = setTimeout(() => { + scheduledTimer = null; + executeHealthCheckCycle().catch((err) => { + healthLog.error(`Initial health selftest error: ${err.message}`); + }); + }, 1000); + + // Schedule recurring selftest every minute + scheduledInterval = setInterval(() => { + executeHealthCheckCycle().catch((err) => { + healthLog.error(`Recurring health selftest error: ${err.message}`); + }); + }, intervalMs); + + healthLog.info(`Health monitor service started (selftest every ${Math.round(intervalMs / 1000)}s via TCP).`); +} + +// MARK: stops the health check service and closes Redis client +export async function stopHealthMonitorService() { + isServiceRunning = false; + if (scheduledTimer) { + clearTimeout(scheduledTimer); + scheduledTimer = null; + } + if (scheduledInterval) { + clearInterval(scheduledInterval); + scheduledInterval = null; + } + + if (redisClient) { + try { + await redisClient.quit(); + } catch { + redisClient.disconnect(); + } finally { + redisClient = null; + } + } + + healthLog.info('Health monitor service stopped.'); +} + +// MARK: returns health monitor status +export function getHealthMonitorStatus() { + return { + enabled: env.healthMonitor?.enabled ?? true, + isRunning: isServiceRunning, + intervalMs: env.healthMonitor?.intervalMs || 60_000, + redisKey: env.redis.key, + redisHost: env.redis.host, + redisPort: env.redis.port, + }; +} +// #endregion