seb/420moni
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

u

Browse Source
This commit is contained in:
seb
2026-07-07 20:02:01 +02:00
parent 80b1d2e147
commit 43475ebff3
9 changed files with 221 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

262
monitor.js
View File

@@ -1,3 +1,4 @@
import 'dotenv/config';
import Database from 'better-sqlite3';
import chalk from 'chalk';
import express from 'express';
@@ -9,15 +10,30 @@ import { dirname } from 'node:path';
// ── Config ────────────────────────────────────────────────────────────────────
const __dirname = dirname(fileURLToPath(import.meta.url));
const TSE_URL = 'http://3.78.227.48:20001/';
const POLL_MS = 60_000; // poll every 60 s
const DB_PATH = './tse_events.db';
const DB_PATH = './monitor.db';
const PORT = 20080;
const POLL_S = 60; // expected interval (gap detection)
const GAP_S = POLL_S * 2; // gap > 120 s → outage
const AGG_INTERVAL = 2 * 60_000; // re-aggregate every 2 min
const AGG_LOOKBACK_H = 3; // hours covered on incremental run
const ENDPOINTS = [];
for (const [key, value] of Object.entries(process.env)) {
if (key.startsWith('ENDPOINT_') && key.endsWith('_URL')) {
const id = key.substring(9, key.length - 4);
ENDPOINTS.push({
id,
url: value,
expectJsonAttr: process.env[`ENDPOINT_${id}_EXPECT_JSON_ATTR`],
expectSubstring: process.env[`ENDPOINT_${id}_EXPECT_SUBSTRING`]
});
}
}
if (ENDPOINTS.length === 0) {
ENDPOINTS.push({ id: 'DEFAULT', url: process.env.TSE_URL || 'http://3.78.227.48:20001/' });
}
// ── DB ────────────────────────────────────────────────────────────────────────
const db = new Database(DB_PATH);
db.pragma('journal_mode = WAL');
@@ -25,72 +41,63 @@ db.pragma('journal_mode = WAL');
db.exec(`
CREATE TABLE IF NOT EXISTS events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
endpoint_id TEXT NOT NULL,
ts TEXT NOT NULL,
status TEXT,
service TEXT,
version TEXT,
serial_number TEXT,
signature_counter INTEGER,
transaction_counter INTEGER,
registered_clients INTEGER,
initialized INTEGER,
created_at TEXT,
fcc_version TEXT,
db_path TEXT,
delta INTEGER,
response_ms INTEGER
response_ms INTEGER,
error_msg TEXT
);
CREATE TABLE IF NOT EXISTS hourly_stats (
hour TEXT PRIMARY KEY,
endpoint_id TEXT,
hour TEXT,
total_polls INTEGER NOT NULL DEFAULT 0,
avg_ms REAL,
min_ms INTEGER,
max_ms INTEGER,
outage_s INTEGER NOT NULL DEFAULT 0,
uptime_pct REAL
uptime_pct REAL,
PRIMARY KEY (endpoint_id, hour)
);
CREATE TABLE IF NOT EXISTS daily_stats (
date TEXT PRIMARY KEY,
endpoint_id TEXT,
date TEXT,
total_polls INTEGER NOT NULL DEFAULT 0,
avg_ms REAL,
min_ms INTEGER,
max_ms INTEGER,
outage_s INTEGER NOT NULL DEFAULT 0,
uptime_pct REAL
uptime_pct REAL,
PRIMARY KEY (endpoint_id, date)
);
CREATE TABLE IF NOT EXISTS outages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
started_at TEXT NOT NULL UNIQUE,
endpoint_id TEXT NOT NULL,
started_at TEXT NOT NULL,
recovered_at TEXT,
duration_s INTEGER
duration_s INTEGER,
UNIQUE (endpoint_id, started_at)
);
`);
// migrate old DBs missing the column
try { db.exec('ALTER TABLE events ADD COLUMN response_ms INTEGER'); } catch {}
try { db.exec('ALTER TABLE events ADD COLUMN delta INTEGER'); } catch {}
// ── Prepared statements ───────────────────────────────────────────────────────
const insertEvent = db.prepare(`
INSERT INTO events
(ts, status, service, version, serial_number,
signature_counter, transaction_counter, registered_clients,
initialized, created_at, fcc_version, db_path, delta, response_ms)
(endpoint_id, ts, status, response_ms, error_msg)
VALUES
(@ts, @status, @service, @version, @serialNumber,
@signatureCounter, @transactionCounter, @registeredClients,
@initialized, @createdAt, @fccVersion, @dbPath, @delta, @responseMs)
(@endpoint_id, @ts, @status, @responseMs, @errorMsg)
`);
const lastRow = db.prepare('SELECT signature_counter FROM events ORDER BY id DESC LIMIT 1');
const stmts = {
eventsInRange: db.prepare(
'SELECT ts, response_ms FROM events WHERE ts >= ? AND ts < ? ORDER BY ts'
'SELECT ts, response_ms FROM events WHERE endpoint_id = ? AND ts >= ? AND ts < ? ORDER BY ts'
),
gapsFrom: db.prepare(`
WITH ordered AS (
SELECT ts, LAG(ts) OVER (ORDER BY ts) AS prev_ts
FROM events WHERE ts >= ?
FROM events WHERE endpoint_id = ? AND ts >= ?
)
SELECT
strftime('%Y-%m-%dT%H:%M:%SZ', prev_ts, '+' || ? || ' seconds') AS started_at,
@@ -101,28 +108,28 @@ const stmts = {
AND CAST((julianday(ts) - julianday(prev_ts)) * 86400 AS INTEGER) > ?
`),
upsertOutage: db.prepare(`
INSERT INTO outages (started_at, recovered_at, duration_s)
VALUES (@started_at, @recovered_at, @duration_s)
ON CONFLICT(started_at) DO UPDATE SET
INSERT INTO outages (endpoint_id, started_at, recovered_at, duration_s)
VALUES (@endpoint_id, @started_at, @recovered_at, @duration_s)
ON CONFLICT(endpoint_id, started_at) DO UPDATE SET
recovered_at = excluded.recovered_at,
duration_s = excluded.duration_s
`),
outagesOverlap: db.prepare(`
SELECT started_at, recovered_at FROM outages
WHERE started_at < ? AND (recovered_at IS NULL OR recovered_at > ?)
WHERE endpoint_id = ? AND started_at < ? AND (recovered_at IS NULL OR recovered_at > ?)
`),
upsertHourly: db.prepare(`
INSERT INTO hourly_stats (hour, total_polls, avg_ms, min_ms, max_ms, outage_s, uptime_pct)
VALUES (@hour, @total_polls, @avg_ms, @min_ms, @max_ms, @outage_s, @uptime_pct)
ON CONFLICT(hour) DO UPDATE SET
INSERT INTO hourly_stats (endpoint_id, hour, total_polls, avg_ms, min_ms, max_ms, outage_s, uptime_pct)
VALUES (@endpoint_id, @hour, @total_polls, @avg_ms, @min_ms, @max_ms, @outage_s, @uptime_pct)
ON CONFLICT(endpoint_id, hour) DO UPDATE SET
total_polls = excluded.total_polls, avg_ms = excluded.avg_ms,
min_ms = excluded.min_ms, max_ms = excluded.max_ms,
outage_s = excluded.outage_s, uptime_pct = excluded.uptime_pct
`),
upsertDaily: db.prepare(`
INSERT INTO daily_stats (date, total_polls, avg_ms, min_ms, max_ms, outage_s, uptime_pct)
VALUES (@date, @total_polls, @avg_ms, @min_ms, @max_ms, @outage_s, @uptime_pct)
ON CONFLICT(date) DO UPDATE SET
INSERT INTO daily_stats (endpoint_id, date, total_polls, avg_ms, min_ms, max_ms, outage_s, uptime_pct)
VALUES (@endpoint_id, @date, @total_polls, @avg_ms, @min_ms, @max_ms, @outage_s, @uptime_pct)
ON CONFLICT(endpoint_id, date) DO UPDATE SET
total_polls = excluded.total_polls, avg_ms = excluded.avg_ms,
min_ms = excluded.min_ms, max_ms = excluded.max_ms,
outage_s = excluded.outage_s, uptime_pct = excluded.uptime_pct
@@ -137,71 +144,85 @@ const fmt = {
ok: s => chalk.bold.green(s),
warn: s => chalk.bold.yellow(s),
error: s => chalk.bold.red(s),
delta: n => n > 0 ? chalk.bold.magenta(`+${n}`) : chalk.dim('±0'),
counter: n => chalk.bold.yellow(String(n)),
sep: () => chalk.dim('─'.repeat(50)),
};
function logOk(label, data, delta, responseMs) {
function logOk(label, responseMs) {
const rtt = responseMs != null ? chalk.bold.green(`${responseMs} ms`) : chalk.dim('null');
console.log(fmt.sep());
console.log(fmt.ts(new Date().toISOString()), fmt.ok(`[${label}]`),
chalk.bold.white(data.service), chalk.dim(`v${data.version}`), chalk.dim('·'), rtt);
console.log(fmt.label('signatureCounter:'), fmt.counter(data.signatureCounter), fmt.delta(delta));
console.log(fmt.label('transactionCounter:'), fmt.value(data.transactionCounter));
console.log(fmt.label('registeredClients:'), fmt.value(data.registeredClients));
console.log(fmt.label('initialized:'), data.initialized ? fmt.ok('true') : fmt.warn('false'));
console.log(fmt.label('serialNumber:'), chalk.dim((data.serialNumber?.slice(0, 16) ?? '') + '…'));
console.log(fmt.label('dbPath:'), fmt.value(data.dbPath));
console.log(fmt.ts(new Date().toISOString()), fmt.ok(`[${label}]`), chalk.dim('·'), rtt);
}
function logError(err, responseMs) {
function logError(errLabel, responseMs) {
const rtt = responseMs != null ? chalk.bold.red(`${responseMs} ms`) : chalk.dim('null');
console.log(fmt.sep());
console.error(fmt.ts(new Date().toISOString()), fmt.error('[ERROR]'), err.message, chalk.dim('·'), rtt);
console.error(fmt.ts(new Date().toISOString()), fmt.error('[ERROR]'), errLabel, chalk.dim('·'), rtt);
}
// ── Poller ────────────────────────────────────────────────────────────────────
let wasError = false;
let isFirstRun = true;
let wasError = {};
let isFirstRun = {};
async function poll() {
let data, responseMs = null;
ENDPOINTS.forEach(ep => {
wasError[ep.id] = false;
isFirstRun[ep.id] = true;
});
async function pollEndpoint(ep) {
let data = {}, responseMs = null, errorMsg = null;
const t0 = Date.now();
let ok = true;
try {
const res = await fetch(TSE_URL, { signal: AbortSignal.timeout(8_000) });
const res = await fetch(ep.url, { signal: AbortSignal.timeout(8_000) });
if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}`);
data = await res.json();
const text = await res.text();
responseMs = Date.now() - t0;
// Check validation criteria
if (ep.expectSubstring && !text.includes(ep.expectSubstring)) {
throw new Error(`Substring '${ep.expectSubstring}' not found`);
}
// Attempt JSON parse if expectJsonAttr or looks like JSON
if (ep.expectJsonAttr || text.startsWith('{') || text.startsWith('[')) {
try {
data = JSON.parse(text);
if (ep.expectJsonAttr && !(ep.expectJsonAttr in data)) {
throw new Error(`JSON missing attribute '${ep.expectJsonAttr}'`);
}
} catch (e) {
if (ep.expectJsonAttr) throw new Error(`Invalid JSON: ${e.message}`);
}
}
} catch (err) {
ok = false;
errorMsg = err.message;
responseMs = err.name === 'TimeoutError' ? null : Date.now() - t0;
if (!wasError) { logError(err, responseMs); wasError = true; }
return;
if (!wasError[ep.id]) { logError(`${ep.id} ERROR: ${err.message}`, responseMs); wasError[ep.id] = true; }
}
const prev = lastRow.get();
const delta = prev ? (data.signatureCounter - prev.signature_counter) : 0;
insertEvent.run({
endpoint_id: ep.id,
ts: new Date().toISOString(),
status: data.status ?? null,
service: data.service ?? null,
version: data.version ?? null,
serialNumber: data.serialNumber ?? null,
signatureCounter: data.signatureCounter ?? null,
transactionCounter: data.transactionCounter ?? null,
registeredClients: data.registeredClients ?? null,
initialized: data.initialized ? 1 : 0,
createdAt: data.createdAt ?? null,
fccVersion: data.fccVersion ?? null,
dbPath: data.dbPath ?? null,
delta,
status: data.status ?? (ok ? 'ok' : 'error'),
responseMs,
errorMsg
});
if (isFirstRun) { logOk('INIT', data, delta, responseMs); isFirstRun = false; }
else if (wasError) { logOk('RECOVERED', data, delta, responseMs); }
wasError = false;
if (isFirstRun[ep.id]) {
if (ok) logOk(`${ep.id} INIT`, responseMs);
isFirstRun[ep.id] = false;
}
else if (ok && wasError[ep.id]) {
logOk(`${ep.id} RECOVERED`, responseMs);
}
if (ok) wasError[ep.id] = false;
}
function poll() {
for (const ep of ENDPOINTS) {
pollEndpoint(ep);
}
}
// ── Aggregator ────────────────────────────────────────────────────────────────
@@ -215,10 +236,10 @@ function msStats(rows) {
};
}
function outageSecondsInWindow(winStart, winEnd) {
function outageSecondsInWindow(epId, winStart, winEnd) {
const wsMs = new Date(winStart).getTime(), weMs = new Date(winEnd).getTime();
return Math.round(
stmts.outagesOverlap.all(winEnd, winStart).reduce((sum, o) => {
stmts.outagesOverlap.all(epId, winEnd, winStart).reduce((sum, o) => {
const s = Math.max(new Date(o.started_at).getTime(), wsMs);
const e = o.recovered_at ? Math.min(new Date(o.recovered_at).getTime(), weMs) : weMs;
return sum + Math.max(0, (e - s) / 1000);
@@ -226,9 +247,9 @@ function outageSecondsInWindow(winStart, winEnd) {
);
}
function detectOutages(fromTs) {
for (const g of stmts.gapsFrom.all(fromTs, POLL_S, GAP_S)) {
stmts.upsertOutage.run({ started_at: g.started_at, recovered_at: g.recovered_at, duration_s: g.gap_s - POLL_S });
function detectOutages(epId, fromTs) {
for (const g of stmts.gapsFrom.all(epId, fromTs, POLL_S, GAP_S)) {
stmts.upsertOutage.run({ endpoint_id: epId, started_at: g.started_at, recovered_at: g.recovered_at, duration_s: g.gap_s - POLL_S });
}
}
@@ -244,25 +265,27 @@ function* eachDay(fromTs, toTs) {
}
function aggregate(fromTs, toTs = new Date().toISOString()) {
detectOutages(fromTs);
for (const h of eachHour(fromTs, toTs)) {
const hourEnd = new Date(new Date(h).getTime() + 3_600_000).toISOString();
const rows = stmts.eventsInRange.all(h, hourEnd);
if (!rows.length) continue; // no events → no row
const { avg_ms, min_ms, max_ms } = msStats(rows);
const outage_s = outageSecondsInWindow(h, hourEnd);
const uptime_pct = +Math.max(0, Math.min(100, (3600 - outage_s) / 3600 * 100)).toFixed(1);
stmts.upsertHourly.run({ hour: h, total_polls: rows.length, avg_ms, min_ms, max_ms, outage_s, uptime_pct });
}
for (const d of eachDay(fromTs, toTs)) {
const dayStart = d + 'T00:00:00Z';
const dayEnd = new Date(new Date(dayStart).getTime() + 86_400_000).toISOString();
const rows = stmts.eventsInRange.all(dayStart, dayEnd);
if (!rows.length) continue; // no events → no row
const { avg_ms, min_ms, max_ms } = msStats(rows);
const outage_s = outageSecondsInWindow(dayStart, dayEnd);
const uptime_pct = +Math.max(0, Math.min(100, (86400 - outage_s) / 86400 * 100)).toFixed(2);
stmts.upsertDaily.run({ date: d, total_polls: rows.length, avg_ms, min_ms, max_ms, outage_s, uptime_pct });
for (const ep of ENDPOINTS) {
detectOutages(ep.id, fromTs);
for (const h of eachHour(fromTs, toTs)) {
const hourEnd = new Date(new Date(h).getTime() + 3_600_000).toISOString();
const rows = stmts.eventsInRange.all(ep.id, h, hourEnd);
if (!rows.length) continue; // no events → no row
const { avg_ms, min_ms, max_ms } = msStats(rows);
const outage_s = outageSecondsInWindow(ep.id, h, hourEnd);
const uptime_pct = +Math.max(0, Math.min(100, (3600 - outage_s) / 3600 * 100)).toFixed(1);
stmts.upsertHourly.run({ endpoint_id: ep.id, hour: h, total_polls: rows.length, avg_ms, min_ms, max_ms, outage_s, uptime_pct });
}
for (const d of eachDay(fromTs, toTs)) {
const dayStart = d + 'T00:00:00Z';
const dayEnd = new Date(new Date(dayStart).getTime() + 86_400_000).toISOString();
const rows = stmts.eventsInRange.all(ep.id, dayStart, dayEnd);
if (!rows.length) continue; // no events → no row
const { avg_ms, min_ms, max_ms } = msStats(rows);
const outage_s = outageSecondsInWindow(ep.id, dayStart, dayEnd);
const uptime_pct = +Math.max(0, Math.min(100, (86400 - outage_s) / 86400 * 100)).toFixed(2);
stmts.upsertDaily.run({ endpoint_id: ep.id, date: d, total_polls: rows.length, avg_ms, min_ms, max_ms, outage_s, uptime_pct });
}
}
}
@@ -270,37 +293,46 @@ function aggregate(fromTs, toTs = new Date().toISOString()) {
const app = express();
app.use((_, res, next) => { res.setHeader('Access-Control-Allow-Origin', '*'); next(); });
app.get('/api/summary', (_, res) => {
app.get('/api/endpoints', (_, res) => {
res.json(ENDPOINTS.map(e => ({ id: e.id, url: e.url })));
});
app.get('/api/summary', (req, res) => {
const epId = req.query.endpoint || ENDPOINTS[0].id;
const stats = db.prepare(`
SELECT SUM(total_polls) AS total_polls, ROUND(AVG(avg_ms),0) AS avg_ms,
MIN(min_ms) AS min_ms, MAX(max_ms) AS max_ms,
SUM(outage_s) AS total_outage_s, ROUND(AVG(uptime_pct),2) AS avg_uptime_pct
FROM daily_stats
`).get();
const outage_count = db.prepare('SELECT COUNT(*) AS n FROM outages').get().n;
const live = db.prepare('SELECT * FROM events ORDER BY id DESC LIMIT 1').get() ?? null;
FROM daily_stats WHERE endpoint_id = ?
`).get(epId);
const outage_count = db.prepare('SELECT COUNT(*) AS n FROM outages WHERE endpoint_id = ?').get(epId).n;
const live = db.prepare('SELECT * FROM events WHERE endpoint_id = ? ORDER BY id DESC LIMIT 1').get(epId) ?? null;
res.json({ ...stats, outage_count, live });
});
app.get('/api/hourly', (req, res) => {
const epId = req.query.endpoint || ENDPOINTS[0].id;
const days = Math.min(30, Math.max(1, parseInt(req.query.days) || 7));
const from = new Date(Date.now() - days * 86_400_000).toISOString();
res.json(db.prepare('SELECT * FROM hourly_stats WHERE hour >= ? ORDER BY hour').all(from));
res.json(db.prepare('SELECT * FROM hourly_stats WHERE endpoint_id = ? AND hour >= ? ORDER BY hour').all(epId, from));
});
app.get('/api/daily', (req, res) => {
const epId = req.query.endpoint || ENDPOINTS[0].id;
const days = Math.min(90, Math.max(1, parseInt(req.query.days) || 30));
const from = new Date(Date.now() - days * 86_400_000).toISOString().slice(0, 10);
res.json(db.prepare('SELECT * FROM daily_stats WHERE date >= ? ORDER BY date').all(from));
res.json(db.prepare('SELECT * FROM daily_stats WHERE endpoint_id = ? AND date >= ? ORDER BY date').all(epId, from));
});
app.get('/api/outages', (req, res) => {
const epId = req.query.endpoint || ENDPOINTS[0].id;
const limit = Math.min(200, Math.max(1, parseInt(req.query.limit) || 50));
res.json(db.prepare('SELECT * FROM outages ORDER BY started_at DESC LIMIT ?').all(limit));
res.json(db.prepare('SELECT * FROM outages WHERE endpoint_id = ? ORDER BY started_at DESC LIMIT ?').all(epId, limit));
});
app.get('/api/live', (_, res) => {
res.json(db.prepare('SELECT * FROM events ORDER BY id DESC LIMIT 1').get() ?? null);
app.get('/api/live', (req, res) => {
const epId = req.query.endpoint || ENDPOINTS[0].id;
res.json(db.prepare('SELECT * FROM events WHERE endpoint_id = ? ORDER BY id DESC LIMIT 1').get(epId) ?? null);
});
// ── HTTP server (created first so Vite HMR can share it) ─────────────────────
@@ -316,7 +348,7 @@ const vite = await createViteServer({
hmr: { server: httpServer },
allowedHosts: ['moni.sebgreen.net'],
watch: {
ignored: ['**/tse_events.db*', '**/monitor.js', '**/node_modules/**']
ignored: ['**/monitor.db*', '**/tse_events.db*', '**/monitor.js', '**/node_modules/**']
}
},
optimizeDeps: {
@@ -328,7 +360,7 @@ const vite = await createViteServer({
app.use(vite.middlewares); // serves React app + HMR on all non-/api routes
httpServer.listen(PORT, () => {
console.log(chalk.bold.bgCyan.black(' TSE MONITOR ') + ' ' + chalk.dim(TSE_URL));
console.log(chalk.bold.bgCyan.black(' TSE MONITOR ') + ' ' + chalk.dim(ENDPOINTS.map(e => e.url).join(', ')));
console.log(chalk.dim(`DB → ${DB_PATH} | poll every ${POLL_MS / 1000}s`));
console.log(chalk.bold.blue(`Dashboard → http://localhost:${PORT}`));
console.log(fmt.sep());