398 lines
17 KiB
JavaScript
398 lines
17 KiB
JavaScript
import 'dotenv/config';
|
|
import Database from 'better-sqlite3';
|
|
import chalk from 'chalk';
|
|
import express from 'express';
|
|
import react from '@vitejs/plugin-react';
|
|
import { createServer as createHttpServer } from 'node:http';
|
|
import { createServer as createViteServer } from 'vite';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { dirname } from 'node:path';
|
|
|
|
// ── Config ────────────────────────────────────────────────────────────────────
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
const POLL_MS = 60_000; // poll every 60 s
|
|
const DB_PATH = './monitor.db';
|
|
const PORT = 20080;
|
|
const POLL_S = 60; // expected interval (gap detection)
|
|
const GAP_S = POLL_S * 2; // gap > 120 s → outage
|
|
const AGG_INTERVAL = 2 * 60_000; // re-aggregate every 2 min
|
|
const AGG_LOOKBACK_H = 3; // hours covered on incremental run
|
|
|
|
const ENDPOINTS = [];
|
|
for (const [key, value] of Object.entries(process.env)) {
|
|
if (key.startsWith('ENDPOINT_') && key.endsWith('_URL')) {
|
|
const id = key.substring(9, key.length - 4);
|
|
ENDPOINTS.push({
|
|
id,
|
|
name: process.env[`ENDPOINT_${id}_NAME`] || id,
|
|
url: value,
|
|
expectJsonAttr: process.env[`ENDPOINT_${id}_EXPECT_JSON_ATTR`],
|
|
expectSubstring: process.env[`ENDPOINT_${id}_EXPECT_SUBSTRING`]
|
|
});
|
|
}
|
|
}
|
|
if (ENDPOINTS.length === 0) {
|
|
ENDPOINTS.push({ id: 'DEFAULT', name: 'Default Endpoint', url: process.env.TSE_URL || 'http://3.78.227.48:20001/' });
|
|
}
|
|
|
|
// ── DB ────────────────────────────────────────────────────────────────────────
|
|
const db = new Database(DB_PATH);
|
|
db.pragma('journal_mode = WAL');
|
|
|
|
db.exec(`
|
|
CREATE TABLE IF NOT EXISTS events (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
endpoint_id TEXT NOT NULL,
|
|
ts TEXT NOT NULL,
|
|
status TEXT,
|
|
response_ms INTEGER,
|
|
error_msg TEXT
|
|
);
|
|
CREATE TABLE IF NOT EXISTS hourly_stats (
|
|
endpoint_id TEXT,
|
|
hour TEXT,
|
|
total_polls INTEGER NOT NULL DEFAULT 0,
|
|
avg_ms REAL,
|
|
min_ms INTEGER,
|
|
max_ms INTEGER,
|
|
outage_s INTEGER NOT NULL DEFAULT 0,
|
|
uptime_pct REAL,
|
|
PRIMARY KEY (endpoint_id, hour)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS daily_stats (
|
|
endpoint_id TEXT,
|
|
date TEXT,
|
|
total_polls INTEGER NOT NULL DEFAULT 0,
|
|
avg_ms REAL,
|
|
min_ms INTEGER,
|
|
max_ms INTEGER,
|
|
outage_s INTEGER NOT NULL DEFAULT 0,
|
|
uptime_pct REAL,
|
|
PRIMARY KEY (endpoint_id, date)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS outages (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
endpoint_id TEXT NOT NULL,
|
|
started_at TEXT NOT NULL,
|
|
recovered_at TEXT,
|
|
duration_s INTEGER,
|
|
UNIQUE (endpoint_id, started_at)
|
|
);
|
|
`);
|
|
|
|
// migrate old DBs missing the column
|
|
try { db.exec('ALTER TABLE events ADD COLUMN response_ms INTEGER'); } catch {}
|
|
|
|
// ── Prepared statements ───────────────────────────────────────────────────────
|
|
const insertEvent = db.prepare(`
|
|
INSERT INTO events
|
|
(endpoint_id, ts, status, response_ms, error_msg)
|
|
VALUES
|
|
(@endpoint_id, @ts, @status, @responseMs, @errorMsg)
|
|
`);
|
|
|
|
const stmts = {
|
|
eventsInRange: db.prepare(
|
|
'SELECT ts, response_ms FROM events WHERE endpoint_id = ? AND ts >= ? AND ts < ? ORDER BY ts'
|
|
),
|
|
gapsFrom: db.prepare(`
|
|
WITH ordered AS (
|
|
SELECT ts, LAG(ts) OVER (ORDER BY ts) AS prev_ts
|
|
FROM events WHERE endpoint_id = ? AND ts >= ?
|
|
)
|
|
SELECT
|
|
strftime('%Y-%m-%dT%H:%M:%SZ', prev_ts, '+' || ? || ' seconds') AS started_at,
|
|
ts AS recovered_at,
|
|
CAST((julianday(ts) - julianday(prev_ts)) * 86400 AS INTEGER) AS gap_s
|
|
FROM ordered
|
|
WHERE prev_ts IS NOT NULL
|
|
AND CAST((julianday(ts) - julianday(prev_ts)) * 86400 AS INTEGER) > ?
|
|
`),
|
|
upsertOutage: db.prepare(`
|
|
INSERT INTO outages (endpoint_id, started_at, recovered_at, duration_s)
|
|
VALUES (@endpoint_id, @started_at, @recovered_at, @duration_s)
|
|
ON CONFLICT(endpoint_id, started_at) DO UPDATE SET
|
|
recovered_at = excluded.recovered_at,
|
|
duration_s = excluded.duration_s
|
|
`),
|
|
outagesOverlap: db.prepare(`
|
|
SELECT started_at, recovered_at FROM outages
|
|
WHERE endpoint_id = ? AND started_at < ? AND (recovered_at IS NULL OR recovered_at > ?)
|
|
`),
|
|
upsertHourly: db.prepare(`
|
|
INSERT INTO hourly_stats (endpoint_id, hour, total_polls, avg_ms, min_ms, max_ms, outage_s, uptime_pct)
|
|
VALUES (@endpoint_id, @hour, @total_polls, @avg_ms, @min_ms, @max_ms, @outage_s, @uptime_pct)
|
|
ON CONFLICT(endpoint_id, hour) DO UPDATE SET
|
|
total_polls = excluded.total_polls, avg_ms = excluded.avg_ms,
|
|
min_ms = excluded.min_ms, max_ms = excluded.max_ms,
|
|
outage_s = excluded.outage_s, uptime_pct = excluded.uptime_pct
|
|
`),
|
|
upsertDaily: db.prepare(`
|
|
INSERT INTO daily_stats (endpoint_id, date, total_polls, avg_ms, min_ms, max_ms, outage_s, uptime_pct)
|
|
VALUES (@endpoint_id, @date, @total_polls, @avg_ms, @min_ms, @max_ms, @outage_s, @uptime_pct)
|
|
ON CONFLICT(endpoint_id, date) DO UPDATE SET
|
|
total_polls = excluded.total_polls, avg_ms = excluded.avg_ms,
|
|
min_ms = excluded.min_ms, max_ms = excluded.max_ms,
|
|
outage_s = excluded.outage_s, uptime_pct = excluded.uptime_pct
|
|
`),
|
|
};
|
|
|
|
// ── Console helpers ───────────────────────────────────────────────────────────
|
|
const fmt = {
|
|
ts: s => chalk.dim(s),
|
|
label: s => chalk.bold.cyan(s.padEnd(22)),
|
|
value: s => chalk.white(String(s)),
|
|
ok: s => chalk.bold.green(s),
|
|
warn: s => chalk.bold.yellow(s),
|
|
error: s => chalk.bold.red(s),
|
|
sep: () => chalk.dim('─'.repeat(50)),
|
|
};
|
|
|
|
function logOk(label, responseMs) {
|
|
const rtt = responseMs != null ? chalk.bold.green(`${responseMs} ms`) : chalk.dim('null');
|
|
console.log(fmt.sep());
|
|
console.log(fmt.ts(new Date().toISOString()), fmt.ok(`[${label}]`), chalk.dim('·'), rtt);
|
|
}
|
|
|
|
function logError(errLabel, responseMs) {
|
|
const rtt = responseMs != null ? chalk.bold.red(`${responseMs} ms`) : chalk.dim('null');
|
|
console.log(fmt.sep());
|
|
console.error(fmt.ts(new Date().toISOString()), fmt.error('[ERROR]'), errLabel, chalk.dim('·'), rtt);
|
|
}
|
|
|
|
// ── Poller ────────────────────────────────────────────────────────────────────
|
|
let wasError = {};
|
|
let isFirstRun = {};
|
|
|
|
ENDPOINTS.forEach(ep => {
|
|
wasError[ep.id] = false;
|
|
isFirstRun[ep.id] = true;
|
|
});
|
|
|
|
async function pollEndpoint(ep) {
|
|
let data = {}, responseMs = null, errorMsg = null;
|
|
const t0 = Date.now();
|
|
let ok = true;
|
|
try {
|
|
const res = await fetch(ep.url, { signal: AbortSignal.timeout(8_000) });
|
|
if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}`);
|
|
const text = await res.text();
|
|
responseMs = Date.now() - t0;
|
|
|
|
// Check validation criteria
|
|
if (ep.expectSubstring && !text.includes(ep.expectSubstring)) {
|
|
throw new Error(`Substring '${ep.expectSubstring}' not found`);
|
|
}
|
|
|
|
// Attempt JSON parse if expectJsonAttr or looks like JSON
|
|
if (ep.expectJsonAttr || text.startsWith('{') || text.startsWith('[')) {
|
|
try {
|
|
data = JSON.parse(text);
|
|
if (ep.expectJsonAttr && !(ep.expectJsonAttr in data)) {
|
|
throw new Error(`JSON missing attribute '${ep.expectJsonAttr}'`);
|
|
}
|
|
} catch (e) {
|
|
if (ep.expectJsonAttr) throw new Error(`Invalid JSON: ${e.message}`);
|
|
}
|
|
}
|
|
} catch (err) {
|
|
ok = false;
|
|
errorMsg = err.message;
|
|
responseMs = err.name === 'TimeoutError' ? null : Date.now() - t0;
|
|
if (!wasError[ep.id]) { logError(`${ep.id} ERROR: ${err.message}`, responseMs); wasError[ep.id] = true; }
|
|
}
|
|
|
|
insertEvent.run({
|
|
endpoint_id: ep.id,
|
|
ts: new Date().toISOString(),
|
|
status: data.status ?? (ok ? 'ok' : 'error'),
|
|
responseMs,
|
|
errorMsg
|
|
});
|
|
|
|
if (isFirstRun[ep.id]) {
|
|
if (ok) logOk(`${ep.id} INIT`, responseMs);
|
|
isFirstRun[ep.id] = false;
|
|
}
|
|
else if (ok && wasError[ep.id]) {
|
|
logOk(`${ep.id} RECOVERED`, responseMs);
|
|
}
|
|
if (ok) wasError[ep.id] = false;
|
|
}
|
|
|
|
function poll() {
|
|
for (const ep of ENDPOINTS) {
|
|
pollEndpoint(ep);
|
|
}
|
|
}
|
|
|
|
// ── Aggregator ────────────────────────────────────────────────────────────────
|
|
function msStats(rows) {
|
|
const vals = rows.map(r => r.response_ms).filter(v => v != null);
|
|
if (!vals.length) return { avg_ms: null, min_ms: null, max_ms: null };
|
|
return {
|
|
avg_ms: Math.round(vals.reduce((a, b) => a + b, 0) / vals.length),
|
|
min_ms: Math.min(...vals),
|
|
max_ms: Math.max(...vals),
|
|
};
|
|
}
|
|
|
|
function outageSecondsInWindow(epId, winStart, winEnd) {
|
|
const wsMs = new Date(winStart).getTime(), weMs = new Date(winEnd).getTime();
|
|
return Math.round(
|
|
stmts.outagesOverlap.all(epId, winEnd, winStart).reduce((sum, o) => {
|
|
const s = Math.max(new Date(o.started_at).getTime(), wsMs);
|
|
const e = o.recovered_at ? Math.min(new Date(o.recovered_at).getTime(), weMs) : weMs;
|
|
return sum + Math.max(0, (e - s) / 1000);
|
|
}, 0)
|
|
);
|
|
}
|
|
|
|
function detectOutages(epId, fromTs) {
|
|
for (const g of stmts.gapsFrom.all(epId, fromTs, POLL_S, GAP_S)) {
|
|
stmts.upsertOutage.run({ endpoint_id: epId, started_at: g.started_at, recovered_at: g.recovered_at, duration_s: g.gap_s - POLL_S });
|
|
}
|
|
}
|
|
|
|
function* eachHour(fromTs, toTs) {
|
|
const cur = new Date(fromTs); cur.setUTCMinutes(0, 0, 0);
|
|
const end = new Date(toTs);
|
|
while (cur <= end) { yield cur.toISOString(); cur.setUTCHours(cur.getUTCHours() + 1); }
|
|
}
|
|
function* eachDay(fromTs, toTs) {
|
|
const cur = new Date(fromTs); cur.setUTCHours(0, 0, 0, 0);
|
|
const end = new Date(toTs);
|
|
while (cur <= end) { yield cur.toISOString().slice(0, 10); cur.setUTCDate(cur.getUTCDate() + 1); }
|
|
}
|
|
|
|
function aggregate(fromTs, toTs = new Date().toISOString()) {
|
|
for (const ep of ENDPOINTS) {
|
|
detectOutages(ep.id, fromTs);
|
|
for (const h of eachHour(fromTs, toTs)) {
|
|
const hourEnd = new Date(new Date(h).getTime() + 3_600_000).toISOString();
|
|
const rows = stmts.eventsInRange.all(ep.id, h, hourEnd);
|
|
if (!rows.length) continue; // no events → no row
|
|
const { avg_ms, min_ms, max_ms } = msStats(rows);
|
|
const outage_s = outageSecondsInWindow(ep.id, h, hourEnd);
|
|
const uptime_pct = +Math.max(0, Math.min(100, (3600 - outage_s) / 3600 * 100)).toFixed(1);
|
|
stmts.upsertHourly.run({ endpoint_id: ep.id, hour: h, total_polls: rows.length, avg_ms, min_ms, max_ms, outage_s, uptime_pct });
|
|
}
|
|
for (const d of eachDay(fromTs, toTs)) {
|
|
const dayStart = d + 'T00:00:00Z';
|
|
const dayEnd = new Date(new Date(dayStart).getTime() + 86_400_000).toISOString();
|
|
const rows = stmts.eventsInRange.all(ep.id, dayStart, dayEnd);
|
|
if (!rows.length) continue; // no events → no row
|
|
const { avg_ms, min_ms, max_ms } = msStats(rows);
|
|
const outage_s = outageSecondsInWindow(ep.id, dayStart, dayEnd);
|
|
const uptime_pct = +Math.max(0, Math.min(100, (86400 - outage_s) / 86400 * 100)).toFixed(2);
|
|
stmts.upsertDaily.run({ endpoint_id: ep.id, date: d, total_polls: rows.length, avg_ms, min_ms, max_ms, outage_s, uptime_pct });
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── Express API ───────────────────────────────────────────────────────────────
|
|
const app = express();
|
|
app.use((_, res, next) => { res.setHeader('Access-Control-Allow-Origin', '*'); next(); });
|
|
|
|
app.get('/api/endpoints', (_, res) => {
|
|
res.json(ENDPOINTS.map(e => ({ id: e.id, name: e.name, url: e.url })));
|
|
});
|
|
|
|
app.get('/api/summary', (req, res) => {
|
|
const epId = req.query.endpoint || ENDPOINTS[0].id;
|
|
const stats = db.prepare(`
|
|
SELECT SUM(total_polls) AS total_polls, ROUND(AVG(avg_ms),0) AS avg_ms,
|
|
MIN(min_ms) AS min_ms, MAX(max_ms) AS max_ms,
|
|
SUM(outage_s) AS total_outage_s, ROUND(AVG(uptime_pct),2) AS avg_uptime_pct
|
|
FROM daily_stats WHERE endpoint_id = ?
|
|
`).get(epId);
|
|
const outage_count = db.prepare('SELECT COUNT(*) AS n FROM outages WHERE endpoint_id = ?').get(epId).n;
|
|
const live = db.prepare('SELECT * FROM events WHERE endpoint_id = ? ORDER BY id DESC LIMIT 1').get(epId) ?? null;
|
|
res.json({ ...stats, outage_count, live });
|
|
});
|
|
|
|
app.get('/api/hourly', (req, res) => {
|
|
const epId = req.query.endpoint || ENDPOINTS[0].id;
|
|
const days = Math.min(30, Math.max(1, parseInt(req.query.days) || 7));
|
|
const from = new Date(Date.now() - days * 86_400_000).toISOString();
|
|
res.json(db.prepare('SELECT * FROM hourly_stats WHERE endpoint_id = ? AND hour >= ? ORDER BY hour').all(epId, from));
|
|
});
|
|
|
|
app.get('/api/daily', (req, res) => {
|
|
const epId = req.query.endpoint || ENDPOINTS[0].id;
|
|
const days = Math.min(90, Math.max(1, parseInt(req.query.days) || 30));
|
|
const from = new Date(Date.now() - days * 86_400_000).toISOString().slice(0, 10);
|
|
res.json(db.prepare('SELECT * FROM daily_stats WHERE endpoint_id = ? AND date >= ? ORDER BY date').all(epId, from));
|
|
});
|
|
|
|
app.get('/api/outages', (req, res) => {
|
|
const epId = req.query.endpoint || ENDPOINTS[0].id;
|
|
const limit = Math.min(200, Math.max(1, parseInt(req.query.limit) || 50));
|
|
res.json(db.prepare('SELECT * FROM outages WHERE endpoint_id = ? ORDER BY started_at DESC LIMIT ?').all(epId, limit));
|
|
});
|
|
|
|
app.get('/api/live', (req, res) => {
|
|
const epId = req.query.endpoint || ENDPOINTS[0].id;
|
|
res.json(db.prepare('SELECT * FROM events WHERE endpoint_id = ? ORDER BY id DESC LIMIT 1').get(epId) ?? null);
|
|
});
|
|
|
|
// ── HTTP server (created first so Vite HMR can share it) ─────────────────────
|
|
const httpServer = createHttpServer(app);
|
|
|
|
// ── Vite dev server (programmatic, middleware mode) ───────────────────────────
|
|
const vite = await createViteServer({
|
|
root: __dirname,
|
|
configFile: false,
|
|
plugins: [react()],
|
|
server: {
|
|
middlewareMode: true,
|
|
hmr: { server: httpServer },
|
|
allowedHosts: ['moni.sebgreen.net'],
|
|
watch: {
|
|
ignored: ['**/monitor.db*', '**/tse_events.db*', '**/monitor.js', '**/node_modules/**']
|
|
}
|
|
},
|
|
optimizeDeps: {
|
|
include: ['react', 'react-dom', '@mui/material', '@mui/icons-material', 'recharts', '@emotion/react', '@emotion/styled']
|
|
},
|
|
appType: 'spa',
|
|
});
|
|
|
|
app.use(vite.middlewares); // serves React app + HMR on all non-/api routes
|
|
|
|
httpServer.listen(PORT, () => {
|
|
console.log(chalk.bold.bgCyan.black(' TSE MONITOR ') + ' ' + chalk.dim(ENDPOINTS.map(e => e.url).join(', ')));
|
|
console.log(chalk.dim(`DB → ${DB_PATH} | poll every ${POLL_MS / 1000}s`));
|
|
console.log(chalk.bold.blue(`Dashboard → http://localhost:${PORT}`));
|
|
console.log(fmt.sep());
|
|
});
|
|
|
|
// ── Aggregation — backfill then incremental ───────────────────────────────────
|
|
{
|
|
const { v: minTs } = db.prepare('SELECT MIN(ts) AS v FROM events').get() ?? {};
|
|
const { v: maxTs } = db.prepare('SELECT MAX(ts) AS v FROM events').get() ?? {};
|
|
if (minTs) {
|
|
console.log(chalk.dim(`[agg] back-filling ${minTs} → ${maxTs} …`));
|
|
aggregate(minTs, maxTs);
|
|
console.log(chalk.dim('[agg] done'));
|
|
}
|
|
}
|
|
|
|
setInterval(
|
|
() => aggregate(new Date(Date.now() - AGG_LOOKBACK_H * 3_600_000).toISOString()),
|
|
AGG_INTERVAL
|
|
);
|
|
|
|
// ── Poll ──────────────────────────────────────────────────────────────────────
|
|
poll();
|
|
setInterval(poll, POLL_MS);
|
|
|
|
// ── Graceful shutdown ─────────────────────────────────────────────────────────
|
|
process.on('SIGINT', async () => {
|
|
console.log('\n' + chalk.bold.red('Shutting down…'));
|
|
await vite.close();
|
|
httpServer.close();
|
|
db.close();
|
|
process.exit(0);
|
|
});
|