import 'dotenv/config'; import Database from 'better-sqlite3'; import chalk from 'chalk'; import express from 'express'; import react from '@vitejs/plugin-react'; import { createServer as createHttpServer } from 'node:http'; import { createServer as createViteServer } from 'vite'; import { fileURLToPath } from 'node:url'; import { dirname } from 'node:path'; // ── Config ──────────────────────────────────────────────────────────────────── const __dirname = dirname(fileURLToPath(import.meta.url)); const POLL_MS = 60_000; // poll every 60 s const DB_PATH = './monitor.db'; const PORT = 20080; const POLL_S = 60; // expected interval (gap detection) const GAP_S = POLL_S * 2; // gap > 120 s → outage const AGG_INTERVAL = 2 * 60_000; // re-aggregate every 2 min const AGG_LOOKBACK_H = 3; // hours covered on incremental run const ENDPOINTS = []; for (const [key, value] of Object.entries(process.env)) { if (key.startsWith('ENDPOINT_') && key.endsWith('_URL')) { const id = key.substring(9, key.length - 4); ENDPOINTS.push({ id, name: process.env[`ENDPOINT_${id}_NAME`] || id, url: value, expectJsonAttr: process.env[`ENDPOINT_${id}_EXPECT_JSON_ATTR`], expectSubstring: process.env[`ENDPOINT_${id}_EXPECT_SUBSTRING`] }); } } if (ENDPOINTS.length === 0) { ENDPOINTS.push({ id: 'DEFAULT', name: 'Default Endpoint', url: process.env.TSE_URL || 'http://3.78.227.48:20001/' }); } // ── DB ──────────────────────────────────────────────────────────────────────── const db = new Database(DB_PATH); db.pragma('journal_mode = WAL'); db.exec(` CREATE TABLE IF NOT EXISTS events ( id INTEGER PRIMARY KEY AUTOINCREMENT, endpoint_id TEXT NOT NULL, ts TEXT NOT NULL, status TEXT, response_ms INTEGER, error_msg TEXT ); CREATE TABLE IF NOT EXISTS hourly_stats ( endpoint_id TEXT, hour TEXT, total_polls INTEGER NOT NULL DEFAULT 0, avg_ms REAL, min_ms INTEGER, max_ms INTEGER, outage_s INTEGER NOT NULL DEFAULT 0, uptime_pct REAL, PRIMARY KEY (endpoint_id, hour) ); CREATE TABLE IF NOT EXISTS daily_stats ( endpoint_id TEXT, date TEXT, total_polls INTEGER NOT NULL DEFAULT 0, avg_ms REAL, min_ms INTEGER, max_ms INTEGER, outage_s INTEGER NOT NULL DEFAULT 0, uptime_pct REAL, PRIMARY KEY (endpoint_id, date) ); CREATE TABLE IF NOT EXISTS outages ( id INTEGER PRIMARY KEY AUTOINCREMENT, endpoint_id TEXT NOT NULL, started_at TEXT NOT NULL, recovered_at TEXT, duration_s INTEGER, UNIQUE (endpoint_id, started_at) ); `); // migrate old DBs missing the column try { db.exec('ALTER TABLE events ADD COLUMN response_ms INTEGER'); } catch {} // ── Prepared statements ─────────────────────────────────────────────────────── const insertEvent = db.prepare(` INSERT INTO events (endpoint_id, ts, status, response_ms, error_msg) VALUES (@endpoint_id, @ts, @status, @responseMs, @errorMsg) `); const stmts = { eventsInRange: db.prepare( 'SELECT ts, response_ms FROM events WHERE endpoint_id = ? AND ts >= ? AND ts < ? ORDER BY ts' ), gapsFrom: db.prepare(` WITH ordered AS ( SELECT ts, LAG(ts) OVER (ORDER BY ts) AS prev_ts FROM events WHERE endpoint_id = ? AND ts >= ? ) SELECT strftime('%Y-%m-%dT%H:%M:%SZ', prev_ts, '+' || ? || ' seconds') AS started_at, ts AS recovered_at, CAST((julianday(ts) - julianday(prev_ts)) * 86400 AS INTEGER) AS gap_s FROM ordered WHERE prev_ts IS NOT NULL AND CAST((julianday(ts) - julianday(prev_ts)) * 86400 AS INTEGER) > ? `), upsertOutage: db.prepare(` INSERT INTO outages (endpoint_id, started_at, recovered_at, duration_s) VALUES (@endpoint_id, @started_at, @recovered_at, @duration_s) ON CONFLICT(endpoint_id, started_at) DO UPDATE SET recovered_at = excluded.recovered_at, duration_s = excluded.duration_s `), outagesOverlap: db.prepare(` SELECT started_at, recovered_at FROM outages WHERE endpoint_id = ? AND started_at < ? AND (recovered_at IS NULL OR recovered_at > ?) `), upsertHourly: db.prepare(` INSERT INTO hourly_stats (endpoint_id, hour, total_polls, avg_ms, min_ms, max_ms, outage_s, uptime_pct) VALUES (@endpoint_id, @hour, @total_polls, @avg_ms, @min_ms, @max_ms, @outage_s, @uptime_pct) ON CONFLICT(endpoint_id, hour) DO UPDATE SET total_polls = excluded.total_polls, avg_ms = excluded.avg_ms, min_ms = excluded.min_ms, max_ms = excluded.max_ms, outage_s = excluded.outage_s, uptime_pct = excluded.uptime_pct `), upsertDaily: db.prepare(` INSERT INTO daily_stats (endpoint_id, date, total_polls, avg_ms, min_ms, max_ms, outage_s, uptime_pct) VALUES (@endpoint_id, @date, @total_polls, @avg_ms, @min_ms, @max_ms, @outage_s, @uptime_pct) ON CONFLICT(endpoint_id, date) DO UPDATE SET total_polls = excluded.total_polls, avg_ms = excluded.avg_ms, min_ms = excluded.min_ms, max_ms = excluded.max_ms, outage_s = excluded.outage_s, uptime_pct = excluded.uptime_pct `), }; // ── Console helpers ─────────────────────────────────────────────────────────── const fmt = { ts: s => chalk.dim(s), label: s => chalk.bold.cyan(s.padEnd(22)), value: s => chalk.white(String(s)), ok: s => chalk.bold.green(s), warn: s => chalk.bold.yellow(s), error: s => chalk.bold.red(s), sep: () => chalk.dim('─'.repeat(50)), }; function logOk(label, responseMs) { const rtt = responseMs != null ? chalk.bold.green(`${responseMs} ms`) : chalk.dim('null'); console.log(fmt.sep()); console.log(fmt.ts(new Date().toISOString()), fmt.ok(`[${label}]`), chalk.dim('·'), rtt); } function logError(errLabel, responseMs) { const rtt = responseMs != null ? chalk.bold.red(`${responseMs} ms`) : chalk.dim('null'); console.log(fmt.sep()); console.error(fmt.ts(new Date().toISOString()), fmt.error('[ERROR]'), errLabel, chalk.dim('·'), rtt); } // ── Poller ──────────────────────────────────────────────────────────────────── let wasError = {}; let isFirstRun = {}; ENDPOINTS.forEach(ep => { wasError[ep.id] = false; isFirstRun[ep.id] = true; }); async function pollEndpoint(ep) { let data = {}, responseMs = null, errorMsg = null; const t0 = Date.now(); let ok = true; try { const res = await fetch(ep.url, { signal: AbortSignal.timeout(8_000) }); if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}`); const text = await res.text(); responseMs = Date.now() - t0; // Check validation criteria if (ep.expectSubstring && !text.includes(ep.expectSubstring)) { throw new Error(`Substring '${ep.expectSubstring}' not found`); } // Attempt JSON parse if expectJsonAttr or looks like JSON if (ep.expectJsonAttr || text.startsWith('{') || text.startsWith('[')) { try { data = JSON.parse(text); if (ep.expectJsonAttr && !(ep.expectJsonAttr in data)) { throw new Error(`JSON missing attribute '${ep.expectJsonAttr}'`); } } catch (e) { if (ep.expectJsonAttr) throw new Error(`Invalid JSON: ${e.message}`); } } } catch (err) { ok = false; errorMsg = err.message; responseMs = err.name === 'TimeoutError' ? null : Date.now() - t0; if (!wasError[ep.id]) { logError(`${ep.id} ERROR: ${err.message}`, responseMs); wasError[ep.id] = true; } } insertEvent.run({ endpoint_id: ep.id, ts: new Date().toISOString(), status: data.status ?? (ok ? 'ok' : 'error'), responseMs, errorMsg }); if (isFirstRun[ep.id]) { if (ok) logOk(`${ep.id} INIT`, responseMs); isFirstRun[ep.id] = false; } else if (ok && wasError[ep.id]) { logOk(`${ep.id} RECOVERED`, responseMs); } if (ok) wasError[ep.id] = false; } function poll() { for (const ep of ENDPOINTS) { pollEndpoint(ep); } } // ── Aggregator ──────────────────────────────────────────────────────────────── function msStats(rows) { const vals = rows.map(r => r.response_ms).filter(v => v != null); if (!vals.length) return { avg_ms: null, min_ms: null, max_ms: null }; return { avg_ms: Math.round(vals.reduce((a, b) => a + b, 0) / vals.length), min_ms: Math.min(...vals), max_ms: Math.max(...vals), }; } function outageSecondsInWindow(epId, winStart, winEnd) { const wsMs = new Date(winStart).getTime(), weMs = new Date(winEnd).getTime(); return Math.round( stmts.outagesOverlap.all(epId, winEnd, winStart).reduce((sum, o) => { const s = Math.max(new Date(o.started_at).getTime(), wsMs); const e = o.recovered_at ? Math.min(new Date(o.recovered_at).getTime(), weMs) : weMs; return sum + Math.max(0, (e - s) / 1000); }, 0) ); } function detectOutages(epId, fromTs) { for (const g of stmts.gapsFrom.all(epId, fromTs, POLL_S, GAP_S)) { stmts.upsertOutage.run({ endpoint_id: epId, started_at: g.started_at, recovered_at: g.recovered_at, duration_s: g.gap_s - POLL_S }); } } function* eachHour(fromTs, toTs) { const cur = new Date(fromTs); cur.setUTCMinutes(0, 0, 0); const end = new Date(toTs); while (cur <= end) { yield cur.toISOString(); cur.setUTCHours(cur.getUTCHours() + 1); } } function* eachDay(fromTs, toTs) { const cur = new Date(fromTs); cur.setUTCHours(0, 0, 0, 0); const end = new Date(toTs); while (cur <= end) { yield cur.toISOString().slice(0, 10); cur.setUTCDate(cur.getUTCDate() + 1); } } function aggregate(fromTs, toTs = new Date().toISOString()) { for (const ep of ENDPOINTS) { detectOutages(ep.id, fromTs); for (const h of eachHour(fromTs, toTs)) { const hourEnd = new Date(new Date(h).getTime() + 3_600_000).toISOString(); const rows = stmts.eventsInRange.all(ep.id, h, hourEnd); if (!rows.length) continue; // no events → no row const { avg_ms, min_ms, max_ms } = msStats(rows); const outage_s = outageSecondsInWindow(ep.id, h, hourEnd); const uptime_pct = +Math.max(0, Math.min(100, (3600 - outage_s) / 3600 * 100)).toFixed(1); stmts.upsertHourly.run({ endpoint_id: ep.id, hour: h, total_polls: rows.length, avg_ms, min_ms, max_ms, outage_s, uptime_pct }); } for (const d of eachDay(fromTs, toTs)) { const dayStart = d + 'T00:00:00Z'; const dayEnd = new Date(new Date(dayStart).getTime() + 86_400_000).toISOString(); const rows = stmts.eventsInRange.all(ep.id, dayStart, dayEnd); if (!rows.length) continue; // no events → no row const { avg_ms, min_ms, max_ms } = msStats(rows); const outage_s = outageSecondsInWindow(ep.id, dayStart, dayEnd); const uptime_pct = +Math.max(0, Math.min(100, (86400 - outage_s) / 86400 * 100)).toFixed(2); stmts.upsertDaily.run({ endpoint_id: ep.id, date: d, total_polls: rows.length, avg_ms, min_ms, max_ms, outage_s, uptime_pct }); } } } // ── Express API ─────────────────────────────────────────────────────────────── const app = express(); app.use((_, res, next) => { res.setHeader('Access-Control-Allow-Origin', '*'); next(); }); app.get('/api/endpoints', (_, res) => { res.json(ENDPOINTS.map(e => ({ id: e.id, name: e.name, url: e.url }))); }); app.get('/api/summary', (req, res) => { const epId = req.query.endpoint || ENDPOINTS[0].id; const stats = db.prepare(` SELECT SUM(total_polls) AS total_polls, ROUND(AVG(avg_ms),0) AS avg_ms, MIN(min_ms) AS min_ms, MAX(max_ms) AS max_ms, SUM(outage_s) AS total_outage_s, ROUND(AVG(uptime_pct),2) AS avg_uptime_pct FROM daily_stats WHERE endpoint_id = ? `).get(epId); const outage_count = db.prepare('SELECT COUNT(*) AS n FROM outages WHERE endpoint_id = ?').get(epId).n; const live = db.prepare('SELECT * FROM events WHERE endpoint_id = ? ORDER BY id DESC LIMIT 1').get(epId) ?? null; res.json({ ...stats, outage_count, live }); }); app.get('/api/hourly', (req, res) => { const epId = req.query.endpoint || ENDPOINTS[0].id; const days = Math.min(30, Math.max(1, parseInt(req.query.days) || 7)); const from = new Date(Date.now() - days * 86_400_000).toISOString(); res.json(db.prepare('SELECT * FROM hourly_stats WHERE endpoint_id = ? AND hour >= ? ORDER BY hour').all(epId, from)); }); app.get('/api/daily', (req, res) => { const epId = req.query.endpoint || ENDPOINTS[0].id; const days = Math.min(90, Math.max(1, parseInt(req.query.days) || 30)); const from = new Date(Date.now() - days * 86_400_000).toISOString().slice(0, 10); res.json(db.prepare('SELECT * FROM daily_stats WHERE endpoint_id = ? AND date >= ? ORDER BY date').all(epId, from)); }); app.get('/api/outages', (req, res) => { const epId = req.query.endpoint || ENDPOINTS[0].id; const limit = Math.min(200, Math.max(1, parseInt(req.query.limit) || 50)); res.json(db.prepare('SELECT * FROM outages WHERE endpoint_id = ? ORDER BY started_at DESC LIMIT ?').all(epId, limit)); }); app.get('/api/live', (req, res) => { const epId = req.query.endpoint || ENDPOINTS[0].id; res.json(db.prepare('SELECT * FROM events WHERE endpoint_id = ? ORDER BY id DESC LIMIT 1').get(epId) ?? null); }); // ── HTTP server (created first so Vite HMR can share it) ───────────────────── const httpServer = createHttpServer(app); // ── Vite dev server (programmatic, middleware mode) ─────────────────────────── const vite = await createViteServer({ root: __dirname, configFile: false, plugins: [react()], server: { middlewareMode: true, hmr: { server: httpServer }, allowedHosts: ['moni.sebgreen.net'], watch: { ignored: ['**/monitor.db*', '**/tse_events.db*', '**/monitor.js', '**/node_modules/**'] } }, optimizeDeps: { include: ['react', 'react-dom', '@mui/material', '@mui/icons-material', 'recharts', '@emotion/react', '@emotion/styled'] }, appType: 'spa', }); app.use(vite.middlewares); // serves React app + HMR on all non-/api routes httpServer.listen(PORT, () => { console.log(chalk.bold.bgCyan.black(' TSE MONITOR ') + ' ' + chalk.dim(ENDPOINTS.map(e => e.url).join(', '))); console.log(chalk.dim(`DB → ${DB_PATH} | poll every ${POLL_MS / 1000}s`)); console.log(chalk.bold.blue(`Dashboard → http://localhost:${PORT}`)); console.log(fmt.sep()); }); // ── Aggregation — backfill then incremental ─────────────────────────────────── { const { v: minTs } = db.prepare('SELECT MIN(ts) AS v FROM events').get() ?? {}; const { v: maxTs } = db.prepare('SELECT MAX(ts) AS v FROM events').get() ?? {}; if (minTs) { console.log(chalk.dim(`[agg] back-filling ${minTs} → ${maxTs} …`)); aggregate(minTs, maxTs); console.log(chalk.dim('[agg] done')); } } setInterval( () => aggregate(new Date(Date.now() - AGG_LOOKBACK_H * 3_600_000).toISOString()), AGG_INTERVAL ); // ── Poll ────────────────────────────────────────────────────────────────────── poll(); setInterval(poll, POLL_MS); // ── Graceful shutdown ───────────────────────────────────────────────────────── process.on('SIGINT', async () => { console.log('\n' + chalk.bold.red('Shutting down…')); await vite.close(); httpServer.close(); db.close(); process.exit(0); });