diff --git a/.env.example b/.env.example index ee5fbf7..2276f1b 100644 --- a/.env.example +++ b/.env.example @@ -44,4 +44,11 @@ OPENROUTER_FREE_KEY=sk-or-v1-3d3c4e OPENROUTER_FREE_MODEL=openrouter/free OPENROUTER_PAID_MODEL=openai/gpt-6-luna:floor +# Backup Service +# BACKUP_ENABLED=true +# BACKUP_DIR=./data/backups +# BACKUP_INTERVAL_MS=3600000 +# BACKUP_HOURLY_RETENTION=50 +# BACKUP_DAILY_RETENTION=50 + diff --git a/docs/api/README.md b/docs/api/README.md index 4937332..ce02e66 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -88,6 +88,8 @@ Generated from `@openapi` annotations in [`src/api/routes/`](../../src/api/route | [`routes/auth/logout.md`](routes/auth/logout.md) | `POST /auth/logout` | | [`routes/auth/me.md`](routes/auth/me.md) | `GET /auth/me` | | [`routes/auth/selected-company.md`](routes/auth/selected-company.md) | `PUT /auth/selected-company` | +| [`routes/backup.md`](routes/backup.md) | `GET /backup` | +| [`routes/backup/run.md`](routes/backup/run.md) | `POST /backup/run` | | [`routes/brand/list.md`](routes/brand/list.md) | `GET /brand` | | [`routes/brand/create.md`](routes/brand/create.md) | `POST /brand` | | [`routes/brand/delete.md`](routes/brand/delete.md) | `DELETE /brand/{id}` | diff --git a/docs/api/routes-all.md b/docs/api/routes-all.md index 4d00a4c..7ab46c8 100644 --- a/docs/api/routes-all.md +++ b/docs/api/routes-all.md @@ -1,6 +1,6 @@ # Dash API Routes Overview (All) -Overview of all 229 API endpoints across all modules. +Overview of all 231 API endpoints across all modules. ### Notation @@ -249,6 +249,15 @@ _none_ - Params: `body: { companyId: int }` - Returns: `200 { user: T41 }` +### Section: /backup + +#### List backups and service status — `GET /backup` +_none_ +- Returns: `200 { status: {}, backups: {} }` +#### Trigger manual backup — `POST /backup/run` +_none_ +- Returns: `200 {}` + ### Section: /brand #### List brands — `GET /brand` diff --git a/docs/api/routes.md b/docs/api/routes.md index d5b7dca..8a907a6 100644 --- a/docs/api/routes.md +++ b/docs/api/routes.md @@ -1,6 +1,6 @@ # Dash API Routes Overview -Overview of all 182 API endpoints across all modules. +Overview of all 184 API endpoints across all modules. ### Notation @@ -232,6 +232,15 @@ _none_ _none_ - Returns: `200 { message: str }` +### Section: /backup + +#### List backups and service status — `GET /backup` +_none_ +- Returns: `200 { status: {}, backups: {} }` +#### Trigger manual backup — `POST /backup/run` +_none_ +- Returns: `200 {}` + ### Section: /brand #### List brands — `GET /brand` diff --git a/docs/api/routes/backup.md b/docs/api/routes/backup.md new file mode 100644 index 0000000..312b67a --- /dev/null +++ b/docs/api/routes/backup.md @@ -0,0 +1,28 @@ +# `GET /backup` + +Protected. Requires a valid session cookie (`credentials: 'include'`). + +List backups and service status + +Retrieves the current status of the internal backup service and lists available hourly, daily, and monthly backups. + +## Response `200` + +Backup service status and listing + +```json +{ + "status": {}, + "backups": {} +} +``` + +## Errors + +| Status | Description | +|--------|-------------| +| 500 | Internal error | + +--- + +_Generated from `@openapi` JSDoc. Do not edit by hand — run `npm run docs:api`._ diff --git a/docs/api/routes/backup/run.md b/docs/api/routes/backup/run.md new file mode 100644 index 0000000..0f7a801 --- /dev/null +++ b/docs/api/routes/backup/run.md @@ -0,0 +1,25 @@ +# `POST /backup/run` + +Protected. Requires a valid session cookie (`credentials: 'include'`). + +Trigger manual backup + +Immediately triggers an on-demand backup snapshot of VFS, storage, and SQLite database. + +## Response `200` + +Backup triggered successfully + +```json +{} +``` + +## Errors + +| Status | Description | +|--------|-------------| +| 500 | Backup execution failure | + +--- + +_Generated from `@openapi` JSDoc. Do not edit by hand — run `npm run docs:api`._ diff --git a/mcp.md b/mcp.md index 64d87f2..462faaa 100644 --- a/mcp.md +++ b/mcp.md @@ -6,12 +6,12 @@ This document explains the architecture, database schema, OAuth 2.1 authenticati ## 1. Overview & Purpose -The Dashback backend provides an MCP server running over **Streamable HTTP** at `/mcp`. It exposes the internal **`dashback_request`** tool (the same tool provided to the local agent via `src/tools/dashback-request.js`) to external MCP clients such as: +The Dashback backend provides an MCP server running over **Streamable HTTP** at `/mcp`. It exposes internal tools (`dashback_request`, `dash-subagent`, and `search-articles`) to external MCP clients such as: - Visual Studio Code / GitHub Copilot - Claude Desktop - Cursor / Windsurf -All operations executed via `dashback_request` are cryptographically authenticated and scoped to the authorizing Dashback user (`google_auth_id`) and their selected company. +All operations executed via these tools are cryptographically authenticated and scoped to the authorizing Dashback user (`google_auth_id`) and their selected company. --- @@ -176,9 +176,12 @@ The client connects to `/mcp` with `Authorization: Bearer `. --- -## 5. The Exposed Tool: `dashback_request` +## 5. The Exposed Tools -The MCP server registers the exact same tool exposed to the internal local agent (`src/tools/dashback-request.js`): +The MCP server exposes three tools to external clients: + +### 1. `dashback_request` +Performs HTTP requests against the Dashback API: ```javascript server.registerTool('dashback_request', { @@ -191,14 +194,44 @@ server.registerTool('dashback_request', { body: z.union([z.string(), z.record(z.string(), z.any())]).nullable().optional(), note: z.string().nullable().optional(), }, -}, async ({ method = 'GET', path, query = null, body = null, note }) => { - // Generates a 5-minute JWT signed with env.jwtSecret scoped to the authenticated user's ID - // Performs fetch against http://127.0.0.1:/api - // Returns status and truncated body (up to 20,000 characters) -}); +}, async ({ method = 'GET', path, query = null, body = null, note }) => { ... }); ``` -Because the internal fetch uses a JWT containing the user's `google_auth_id`, all Dashback route-level permissions, company isolation (`req.user.selectedCompanyId`), audit logging, and data scoping apply automatically. +### 2. `dash-subagent` +Delegates a self-contained task to an isolated subagent with access to Dashback tools and VFS: + +```javascript +server.registerTool('dash-subagent', { + title: 'Dashback Subagent', + description: 'Delegate a task to an isolated subagent with access to Dashback tools and VFS.', + inputSchema: { + task: z.string().describe('Specific instructions and goal for the subagent.'), + skills: z.array(z.string()).nullable().optional().describe('Array of SKILL names or slugs to inject.'), + conversation_id: z.number().int().positive().nullable().optional().describe('Existing subagent conversation ID to follow up on, or null to create a new one.'), + max_iterations: z.number().int().min(1).max(30).nullable().optional().describe('Safety bound on tool loops (default 15, max 30).'), + model: z.string().nullable().optional().describe('Optional OpenRouter model override.'), + note: z.string().nullable().optional(), + }, +}, async ({ task, skills, conversation_id, max_iterations, model, note }, extra) => { ... }); +``` + +### 3. `search-articles` +Searches articles across name, brand, EAN, SKU, and supplier SKU (AND logic, ordered by name): + +```javascript +server.registerTool('search-articles', { + title: 'Search Articles', + description: 'Search articles by name, brand, EAN, SKU, or supplier SKU (AND logic, ordered by name).', + inputSchema: { + searchquery: z.string().nullable().optional().describe('Keywords matching name, brand, EAN, SKU, or supplier SKU (AND logic).'), + excludewords: z.string().nullable().optional().describe('Keywords to exclude (AND logic).'), + offset: z.number().int().min(0).nullable().optional().describe('Pagination offset (default 0).'), + limit: z.number().int().min(1).max(20).nullable().optional().describe('Window size 1–20 (default 3).'), + }, +}, async ({ searchquery, excludewords, offset, limit }) => { ... }); +``` + +Because all tool executions use credentials scoped to the authenticated user's `google_auth_id`, all Dashback route-level permissions, company isolation (`req.user.selectedCompanyId`), audit logging, and data scoping apply automatically. --- diff --git a/scripts/test-backup-service.js b/scripts/test-backup-service.js new file mode 100644 index 0000000..c9b2a4d --- /dev/null +++ b/scripts/test-backup-service.js @@ -0,0 +1,153 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import assert from 'node:assert'; +import env from '../src/config/env.js'; +import { + createHourlyBackup, + listBackups, + pruneBackups, + getBackupServiceStatus, + startBackupService, + stopBackupService, + resolveBackupPaths, +} from '../src/services/backupService.js'; +import { closeStore } from '../src/db/store.js'; + +async function runTests() { + console.log('--- Starting Backup Service Verification ---'); + + // 1. Verify path resolution and directories + const paths = resolveBackupPaths(); + console.log(`Backup root dir: ${paths.rootDir}`); + assert(paths.rootDir.endsWith('data/backups'), 'Backup root should be data/backups'); + + // 2. Execute a backup + console.log('\n[Test 1] Running createHourlyBackup()...'); + const backupResult = await createHourlyBackup({ manual: true }); + assert(backupResult.success === true, 'Backup should report success'); + console.log(`Backup created: ${backupResult.id}, size: ${(backupResult.totalSizeBytes / 1024 / 1024).toFixed(2)} MB`); + console.log(`Promoted to daily: ${backupResult.promotedDaily}, Promoted to monthly: ${backupResult.promotedMonthly}`); + + // Verify files in the created hourly backup + const hourlyBackupFolder = path.join(paths.hourlyDir, backupResult.id); + assert(fs.existsSync(hourlyBackupFolder), 'Hourly backup folder should exist'); + assert(fs.existsSync(path.join(hourlyBackupFolder, 'vfs.tar.gz')), 'vfs.tar.gz should exist'); + assert(fs.existsSync(path.join(hourlyBackupFolder, 'storage.tar')), 'storage.tar should exist'); + assert(fs.existsSync(path.join(hourlyBackupFolder, 'manifest.json')), 'manifest.json should exist'); + if (env.dbProvider === 'sqlite') { + const dbName = path.basename(env.sqlite.filename); + assert(fs.existsSync(path.join(hourlyBackupFolder, dbName)), `${dbName} should exist`); + console.log(`Verified SQLite database backup file: ${dbName}`); + } + + // Verify daily backup directory + const { dateKey, monthKey } = backupResult.manifest; + const dailyFolder = path.join(paths.dailyDir, `backup-${dateKey}`); + assert(fs.existsSync(dailyFolder), 'Daily backup folder should exist'); + assert(fs.existsSync(path.join(dailyFolder, 'manifest.json')), 'Daily manifest should exist'); + + // Verify monthly backup directory + const monthlyFolder = path.join(paths.monthlyDir, `backup-${monthKey}`); + assert(fs.existsSync(monthlyFolder), 'Monthly backup folder should exist'); + assert(fs.existsSync(path.join(monthlyFolder, 'manifest.json')), 'Monthly manifest should exist'); + + // 3. Test listBackups and getBackupServiceStatus + console.log('\n[Test 2] Testing listBackups() and getBackupServiceStatus()...'); + const status = getBackupServiceStatus(); + assert(typeof status.enabled === 'boolean', 'status.enabled should be boolean'); + assert(status.retention.hourlyMax === 50, 'hourly retention should be 50'); + assert(status.retention.dailyMax === 50, 'daily retention should be 50'); + + const listing = listBackups(); + assert(listing.hourly.length >= 1, 'Should have at least 1 hourly backup'); + assert(listing.daily.length >= 1, 'Should have at least 1 daily backup'); + assert(listing.monthly.length >= 1, 'Should have at least 1 monthly backup'); + console.log(`Counts -> hourly: ${listing.counts.hourly}, daily: ${listing.counts.daily}, monthly: ${listing.counts.monthly}`); + + // 4. Test Retention Pruning (exceeding 50 entries) + console.log('\n[Test 3] Testing retention pruning for > 50 entries...'); + // Create dummy hourly backup folders + for (let i = 1; i <= 55; i++) { + const dummyName = `backup-2020-01-01_${String(i).padStart(2, '0')}-00-00`; + const dummyPath = path.join(paths.hourlyDir, dummyName); + fs.mkdirSync(dummyPath, { recursive: true }); + fs.writeFileSync(path.join(dummyPath, 'manifest.json'), '{}'); + } + + // Create dummy daily backup folders + for (let i = 1; i <= 55; i++) { + const dummyName = `backup-2020-01-${String(i).padStart(2, '0')}`; + const dummyPath = path.join(paths.dailyDir, dummyName); + fs.mkdirSync(dummyPath, { recursive: true }); + fs.writeFileSync(path.join(dummyPath, 'manifest.json'), '{}'); + } + + // Create dummy monthly backup folders (should never be pruned!) + for (let i = 1; i <= 55; i++) { + const dummyName = `backup-2010-${String(i).padStart(2, '0')}`; + const dummyPath = path.join(paths.monthlyDir, dummyName); + fs.mkdirSync(dummyPath, { recursive: true }); + fs.writeFileSync(path.join(dummyPath, 'manifest.json'), '{}'); + } + + const pruneResult = pruneBackups(); + console.log(`Pruned ${pruneResult.hourlyPruned} hourly, ${pruneResult.dailyPruned} daily`); + assert(pruneResult.hourlyPruned >= 5, 'Should have pruned at least 5 hourly backups'); + assert(pruneResult.dailyPruned >= 5, 'Should have pruned at least 5 daily backups'); + + const listingAfterPrune = listBackups(); + assert(listingAfterPrune.counts.hourly <= 50, `Hourly count should be <= 50, got ${listingAfterPrune.counts.hourly}`); + assert(listingAfterPrune.counts.daily <= 50, `Daily count should be <= 50, got ${listingAfterPrune.counts.daily}`); + assert(listingAfterPrune.counts.monthly >= 55, `Monthly count should NOT be pruned (got ${listingAfterPrune.counts.monthly})`); + console.log('Retention pruning verified: 50 hourly max, 50 daily max, monthly preserved forever.'); + + // Clean up dummy test backups + for (let i = 1; i <= 55; i++) { + const hPath = path.join(paths.hourlyDir, `backup-2020-01-01_${String(i).padStart(2, '0')}-00-00`); + if (fs.existsSync(hPath)) fs.rmSync(hPath, { recursive: true, force: true }); + const dPath = path.join(paths.dailyDir, `backup-2020-01-${String(i).padStart(2, '0')}`); + if (fs.existsSync(dPath)) fs.rmSync(dPath, { recursive: true, force: true }); + const mPath = path.join(paths.monthlyDir, `backup-2010-${String(i).padStart(2, '0')}`); + if (fs.existsSync(mPath)) fs.rmSync(mPath, { recursive: true, force: true }); + } + + // 5. Test MSSQL warning behavior + console.log('\n[Test 4] Testing MSSQL console warning behavior...'); + const originalProvider = env.dbProvider; + let warningCaptured = false; + const origWarn = console.warn; + console.warn = (...args) => { + const text = args.join(' '); + if (text.includes('MSSQL database backup is not implemented')) { + warningCaptured = true; + } + origWarn(...args); + }; + + try { + env.dbProvider = 'mssql'; + const mssqlBackup = await createHourlyBackup({ manual: true }); + assert(warningCaptured, 'Console warning must be printed when running on MSSQL'); + assert(mssqlBackup.manifest.files.database.backedUp === false, 'Database backup must be skipped on MSSQL'); + console.log('MSSQL warning and database skip verified.'); + } finally { + console.warn = origWarn; + env.dbProvider = originalProvider; + } + + // 6. Test Lifecycle methods + console.log('\n[Test 5] Testing startBackupService() and stopBackupService()...'); + startBackupService(); + assert(getBackupServiceStatus().isRunning === true, 'Service should be running'); + await stopBackupService(); + assert(getBackupServiceStatus().isRunning === false, 'Service should be stopped'); + console.log('Service lifecycle start/stop verified.'); + + await closeStore(); + console.log('\n--- ALL BACKUP SERVICE TESTS PASSED ---'); +} + +runTests().catch((err) => { + console.error('Test failed:', err); + process.exit(1); +}); diff --git a/src/api/index.js b/src/api/index.js index 17507ee..661255d 100644 --- a/src/api/index.js +++ b/src/api/index.js @@ -18,6 +18,7 @@ import { getOrCreateCert } from '../util/cert.js'; import { createLogger } from '../util/logger.js'; import v1Router from '../v1/index.js'; import { startBackendMonitor, stopBackendMonitor, isDevMode } from '../backendmonitor/server.js'; +import { startBackupService, stopBackupService } from '../services/backupService.js'; const apiLog = createLogger('api'); @@ -245,6 +246,12 @@ export async function startApi() { } } + try { + startBackupService(); + } catch (err) { + apiLog.error('Failed to start Backup Service:', err.message); + } + return server; })(); @@ -267,6 +274,7 @@ export async function stopApi() { await shutdownImageCropWorkers(); // Same for the pooled sandbox threads that run `javascript_execute` / applets. await shutdownJavascriptSandboxWorkers(); + await stopBackupService(); const closers = []; if (server) { const s = server; diff --git a/src/api/routes/backup.js b/src/api/routes/backup.js new file mode 100644 index 0000000..29fef7d --- /dev/null +++ b/src/api/routes/backup.js @@ -0,0 +1,72 @@ +import { listBackups, getBackupServiceStatus, createHourlyBackup } from '../../services/backupService.js'; + +/** + * Registers internal backup service status and execution endpoints. + * @param {import('express').Application} app - Express app + * @param {string} [basePath=''] - Optional URL prefix for all routes in this module + */ +export function register(app, basePath = '') { + /** + * @openapi + * /backup: + * get: + * tags: [Backup] + * summary: List backups and service status + * description: Retrieves the current status of the internal backup service and lists available hourly, daily, and monthly backups. + * responses: + * 200: + * description: Backup service status and listing + * content: + * application/json: + * schema: + * type: object + * properties: + * status: + * type: object + * backups: + * type: object + * 500: + * description: Internal error + */ + // #region GET /backup + // MARK: handles GET /backup to list backups and service status + app.get(`${basePath}/backup`, async (_req, res) => { + try { + const status = getBackupServiceStatus(); + const backups = listBackups(); + res.status(200).json({ status, backups }); + } catch (error) { + res.status(500).json({ error: error.message }); + } + }); + // #endregion + + /** + * @openapi + * /backup/run: + * post: + * tags: [Backup] + * summary: Trigger manual backup + * description: Immediately triggers an on-demand backup snapshot of VFS, storage, and SQLite database. + * responses: + * 200: + * description: Backup triggered successfully + * content: + * application/json: + * schema: + * type: object + * 500: + * description: Backup execution failure + */ + // #region POST /backup/run + // MARK: handles POST /backup/run to trigger manual backup + app.post(`${basePath}/backup/run`, async (_req, res) => { + try { + const result = await createHourlyBackup({ manual: true }); + res.status(200).json(result); + } catch (error) { + res.status(500).json({ error: error.message }); + } + }); + // #endregion +} diff --git a/src/api/routes/mcp.js b/src/api/routes/mcp.js index 9edf58e..c2b82c2 100644 --- a/src/api/routes/mcp.js +++ b/src/api/routes/mcp.js @@ -9,6 +9,9 @@ import env from '../../config/env.js'; import * as mcpDb from '../../db/mcp.js'; import * as googleAuthDb from '../../db/googleAuth.js'; import { TOOL_DESCRIPTIONS, TOOL_PARAMETER_DESCRIPTIONS } from '../../tools/prompts.js'; +import { createSearchArticlesTool } from '../../tools/search-articles.js'; +import { createSubagentTool } from '../../tools/subagent.js'; +import { resolveDefaultModel } from '../../tools/llm-key.js'; const MAX_RESPONSE_CHARS = 20_000; const REQUEST_TIMEOUT_MS = 30_000; @@ -177,6 +180,89 @@ function createDashbackMcpServer({ user }) { } ); + server.registerTool( + 'dash-subagent', + { + title: 'Dashback Subagent', + description: TOOL_DESCRIPTIONS.subagent || 'Delegate a task to an isolated subagent with access to Dashback tools and VFS.', + inputSchema: { + task: z.string().describe(TOOL_PARAMETER_DESCRIPTIONS.subagent?.task || 'Specific instructions and goal for the subagent.'), + skills: z.array(z.string()).nullable().optional().describe(TOOL_PARAMETER_DESCRIPTIONS.subagent?.skills || 'Array of SKILL names or slugs to inject into the subagent prompt.'), + conversation_id: z.number().int().positive().nullable().optional().describe(TOOL_PARAMETER_DESCRIPTIONS.subagent?.conversation_id || 'Existing subagent conversation ID to follow up on, or null to create a new one.'), + max_iterations: z.number().int().min(1).max(30).nullable().optional().describe(TOOL_PARAMETER_DESCRIPTIONS.subagent?.max_iterations || 'Safety bound on tool loops (default 15, max 30).'), + model: z.string().nullable().optional().describe('Optional OpenRouter model override. Defaults to company default model.'), + note: z.string().nullable().optional().describe(TOOL_PARAMETER_DESCRIPTIONS.subagent?.note || 'Status message shown during execution.'), + }, + }, + async ({ task, skills = null, conversation_id = null, max_iterations = null, model = null, note = null }, extra = {}) => { + const freshUser = await googleAuthDb.getById(user.id); + const companyId = freshUser?.selectedCompanyId || user.selectedCompanyId; + if (!companyId) throw new Error('No company selected'); + + const parentModel = model || (await resolveDefaultModel(user.id)); + const tool = createSubagentTool({ + googleAuthId: user.id, + companyId, + conversationId: null, + parentModel, + depth: 0, + }); + + console.log(`[mcp] Tool invoke dash-subagent: task="${task?.slice(0, 100)}"`); + const signals = [AbortSignal.timeout(300_000)]; + if (extra?.signal) signals.push(extra.signal); + const signal = AbortSignal.any(signals); + + const result = await tool.invoke( + { task, skills, conversation_id, max_iterations, note }, + { signal } + ); + + let text = JSON.stringify(result, null, 2); + if (text.length > MAX_RESPONSE_CHARS) { + text = `${text.slice(0, MAX_RESPONSE_CHARS)}\n…[truncated, ${text.length} chars total]`; + } + return { + content: [{ type: 'text', text }], + }; + } + ); + + server.registerTool( + 'search-articles', + { + title: 'Search Articles', + description: TOOL_DESCRIPTIONS.search_articles || 'Search articles by name, brand, EAN, SKU, or supplier SKU (AND logic, ordered by name).', + inputSchema: { + searchquery: z.string().nullable().optional().describe(TOOL_PARAMETER_DESCRIPTIONS.search_articles?.searchquery || 'Keywords matching name, brand, EAN, SKU, or supplier SKU (AND logic).'), + excludewords: z.string().nullable().optional().describe(TOOL_PARAMETER_DESCRIPTIONS.search_articles?.excludewords || 'Keywords to exclude (AND logic).'), + offset: z.number().int().min(0).nullable().optional().describe(TOOL_PARAMETER_DESCRIPTIONS.search_articles?.offset || 'Pagination offset (default 0).'), + limit: z.number().int().min(1).max(20).nullable().optional().describe(TOOL_PARAMETER_DESCRIPTIONS.search_articles?.limit || 'Window size 1–20 (default 3).'), + }, + }, + async ({ searchquery = '', excludewords = '', offset = 0, limit = 3 } = {}) => { + const freshUser = await googleAuthDb.getById(user.id); + const companyId = freshUser?.selectedCompanyId || user.selectedCompanyId; + if (!companyId) throw new Error('No company selected'); + + console.log(`[mcp] Tool invoke search-articles: query="${searchquery ?? ''}" exclude="${excludewords ?? ''}" offset=${offset} limit=${limit}`); + const tool = createSearchArticlesTool({ googleAuthId: user.id, companyId }); + const result = await tool.invoke({ + searchquery: searchquery ?? '', + excludewords: excludewords ?? '', + offset: offset ?? 0, + limit: limit ?? 3, + }); + let text = JSON.stringify(result, null, 2); + if (text.length > MAX_RESPONSE_CHARS) { + text = `${text.slice(0, MAX_RESPONSE_CHARS)}\n…[truncated, ${text.length} chars total]`; + } + return { + content: [{ type: 'text', text }], + }; + } + ); + return server; } diff --git a/src/config/env.js b/src/config/env.js index fad79a2..efd0c22 100644 --- a/src/config/env.js +++ b/src/config/env.js @@ -45,4 +45,11 @@ export default { encrypt: process.env.MSSQL_ENCRYPT === 'true', trustServerCertificate: process.env.MSSQL_TRUST_SERVER_CERTIFICATE === 'true', }, + backup: { + enabled: process.env.BACKUP_ENABLED !== 'false', + dir: process.env.BACKUP_DIR ? resolve(process.env.BACKUP_DIR) : resolve(__dirname, '../../data/backups'), + intervalMs: Number(process.env.BACKUP_INTERVAL_MS) || 60 * 60 * 1000, + hourlyRetentionCount: Number(process.env.BACKUP_HOURLY_RETENTION) || 50, + dailyRetentionCount: Number(process.env.BACKUP_DAILY_RETENTION) || 50, + }, }; diff --git a/src/services/backupService.js b/src/services/backupService.js new file mode 100644 index 0000000..b8f8850 --- /dev/null +++ b/src/services/backupService.js @@ -0,0 +1,510 @@ +// #region Imports & Configuration +import fs from 'node:fs'; +import path from 'node:path'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import env from '../config/env.js'; +import { getProvider } from '../db/store.js'; +import { createLogger } from '../util/logger.js'; + +const execFileAsync = promisify(execFile); +const backupLog = createLogger('backup'); + +let scheduledTimer = null; +let scheduledInterval = null; +let isServiceRunning = false; +let activeBackupPromise = null; +// #endregion + +// #region Path Helpers +// MARK: resolves standard directories for backups and source assets +export function resolveBackupPaths() { + const rootDir = env.backup?.dir || path.resolve(process.cwd(), 'data/backups'); + const vfsDir = path.resolve(process.cwd(), 'data/vfs'); + const storageDir = path.resolve(process.cwd(), 'data/storage'); + const sqlitePath = env.sqlite?.filename || path.resolve(process.cwd(), 'data/dash.db'); + + return { + rootDir, + hourlyDir: path.join(rootDir, 'hourly'), + dailyDir: path.join(rootDir, 'daily'), + monthlyDir: path.join(rootDir, 'monthly'), + tempDir: path.join(rootDir, '.tmp'), + vfsDir, + storageDir, + sqlitePath, + }; +} + +// MARK: ensures all backup tier directories exist +export function ensureBackupDirectories() { + const paths = resolveBackupPaths(); + for (const dir of [paths.rootDir, paths.hourlyDir, paths.dailyDir, paths.monthlyDir, paths.tempDir]) { + fs.mkdirSync(dir, { recursive: true }); + } + return paths; +} +// #endregion + +// #region Format & Timestamp Helpers +// MARK: formats current Date into zero-padded parts +function formatDateTimeComponents(date = new Date()) { + const pad = (val) => String(val).padStart(2, '0'); + const year = date.getFullYear(); + const month = pad(date.getMonth() + 1); + const day = pad(date.getDate()); + const hours = pad(date.getHours()); + const minutes = pad(date.getMinutes()); + const seconds = pad(date.getSeconds()); + + return { + timestamp: `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`, + dateKey: `${year}-${month}-${day}`, + monthKey: `${year}-${month}`, + isoString: date.toISOString(), + }; +} +// #endregion + +// #region Archiving & Database Snapshotting +// MARK: archives a directory into tar (uncompressed) or tar.gz (compressed) +async function archiveDirectory(sourcePath, destinationArchive, { gzip = false } = {}) { + fs.mkdirSync(path.dirname(destinationArchive), { recursive: true }); + if (fs.existsSync(destinationArchive)) { + fs.unlinkSync(destinationArchive); + } + + const sourceExists = fs.existsSync(sourcePath); + if (!sourceExists) { + // Create an empty archive if source directory does not exist + const flags = gzip ? ['-czf', destinationArchive, '-T', '/dev/null'] : ['-cf', destinationArchive, '-T', '/dev/null']; + await execFileAsync('tar', flags); + return { exists: false, sizeBytes: fs.statSync(destinationArchive).size }; + } + + const parentDir = path.dirname(sourcePath); + const folderName = path.basename(sourcePath); + const flags = gzip ? ['-czf', destinationArchive, '-C', parentDir, folderName] : ['-cf', destinationArchive, '-C', parentDir, folderName]; + + try { + await execFileAsync('tar', flags); + } catch (err) { + // GNU tar returns exit code 1 if files changed while being read; treat as non-fatal + if (err.code === 1 && /file changed as we read it/i.test(err.stderr || '')) { + backupLog.warn(`tar warning: ${err.stderr.trim()}`); + } else { + throw err; + } + } + + const stats = fs.statSync(destinationArchive); + return { exists: true, sizeBytes: stats.size }; +} + +// MARK: creates an atomic SQLite database snapshot file +async function snapshotSqliteDatabase(targetFile) { + if (env.dbProvider === 'mssql') { + const warningMessage = 'MSSQL database backup is not implemented; skipping database backup.'; + console.warn(`[backup] WARNING: ${warningMessage}`); + backupLog.warn(warningMessage); + return { success: false, reason: 'mssql_unsupported', sizeBytes: 0 }; + } + + fs.mkdirSync(path.dirname(targetFile), { recursive: true }); + if (fs.existsSync(targetFile)) { + fs.unlinkSync(targetFile); + } + + try { + const provider = await getProvider(); + if (provider?.db && typeof provider.db.backup === 'function') { + await provider.db.backup(targetFile); + const stats = fs.statSync(targetFile); + return { success: true, sizeBytes: stats.size }; + } + + if (provider?.query) { + const escapedPath = targetFile.replace(/'/g, "''"); + await provider.query(`VACUUM INTO '${escapedPath}'`); + const stats = fs.statSync(targetFile); + return { success: true, sizeBytes: stats.size }; + } + } catch (err) { + backupLog.warn(`Online SQLite backup method failed (${err.message}), falling back to direct copy`); + } + + // Direct file fallback + const { sqlitePath } = resolveBackupPaths(); + if (fs.existsSync(sqlitePath)) { + fs.copyFileSync(sqlitePath, targetFile); + const stats = fs.statSync(targetFile); + return { success: true, sizeBytes: stats.size }; + } + + throw new Error(`SQLite source file not found at ${sqlitePath}`); +} +// #endregion + +// #region Promotion & Retention +// MARK: links or copies all files from one backup folder into another +function mirrorBackupDirectory(sourceDir, targetDir) { + fs.mkdirSync(targetDir, { recursive: true }); + for (const fileName of fs.readdirSync(sourceDir)) { + const srcFile = path.join(sourceDir, fileName); + const destFile = path.join(targetDir, fileName); + if (fs.existsSync(destFile)) { + fs.unlinkSync(destFile); + } + try { + fs.linkSync(srcFile, destFile); + } catch { + fs.copyFileSync(srcFile, destFile); + } + } +} + +// MARK: enforces retention limits for hourly (50) and daily (50) tiers +export function pruneBackups() { + const { hourlyDir, dailyDir } = resolveBackupPaths(); + const hourlyLimit = env.backup?.hourlyRetentionCount || 50; + const dailyLimit = env.backup?.dailyRetentionCount || 50; + + let hourlyPruned = 0; + let dailyPruned = 0; + + // Prune hourly backups: keep last 50 + if (fs.existsSync(hourlyDir)) { + const entries = fs.readdirSync(hourlyDir, { withFileTypes: true }) + .filter((e) => e.isDirectory() && e.name.startsWith('backup-')) + .map((e) => e.name) + .sort(); + + if (entries.length > hourlyLimit) { + const toRemove = entries.slice(0, entries.length - hourlyLimit); + for (const name of toRemove) { + const target = path.join(hourlyDir, name); + fs.rmSync(target, { recursive: true, force: true }); + hourlyPruned++; + } + } + } + + // Prune daily backups: keep last 50 + if (fs.existsSync(dailyDir)) { + const entries = fs.readdirSync(dailyDir, { withFileTypes: true }) + .filter((e) => e.isDirectory() && e.name.startsWith('backup-')) + .map((e) => e.name) + .sort(); + + if (entries.length > dailyLimit) { + const toRemove = entries.slice(0, entries.length - dailyLimit); + for (const name of toRemove) { + const target = path.join(dailyDir, name); + fs.rmSync(target, { recursive: true, force: true }); + dailyPruned++; + } + } + } + + // Monthly backups are kept forever — no pruning is performed on monthlyDir. + + if (hourlyPruned > 0 || dailyPruned > 0) { + backupLog.info(`Pruned backups: ${hourlyPruned} hourly, ${dailyPruned} daily.`); + } + + return { hourlyPruned, dailyPruned }; +} +// #endregion + +// #region Core Backup Execution +// MARK: executes a full backup and promotes to daily/monthly tiers as appropriate +export async function createHourlyBackup({ manual = false } = {}) { + if (activeBackupPromise) { + backupLog.warn('A backup is already in progress, waiting for it to finish...'); + return activeBackupPromise; + } + + activeBackupPromise = (async () => { + const start = Date.now(); + const paths = ensureBackupDirectories(); + const { timestamp, dateKey, monthKey, isoString } = formatDateTimeComponents(new Date()); + + const backupId = `backup-${timestamp}`; + const stagingDir = path.join(paths.tempDir, `${backupId}-${Date.now()}`); + const finalHourlyDir = path.join(paths.hourlyDir, backupId); + + backupLog.info(`Starting backup (${backupId})...`); + fs.mkdirSync(stagingDir, { recursive: true }); + + try { + // 1. VFS -> gzipped tar (vfs.tar.gz) + const vfsResult = await archiveDirectory(paths.vfsDir, path.join(stagingDir, 'vfs.tar.gz'), { gzip: true }); + + // 2. Storage images -> uncompressed tar (storage.tar) + const storageResult = await archiveDirectory(paths.storageDir, path.join(stagingDir, 'storage.tar'), { gzip: false }); + + // 3. SQLite DB -> file (dash.db) + const dbFileName = env.sqlite?.filename ? path.basename(env.sqlite.filename) : 'dash.db'; + const dbResult = await snapshotSqliteDatabase(path.join(stagingDir, dbFileName)); + + // 4. Manifest + const totalSize = (vfsResult.sizeBytes || 0) + (storageResult.sizeBytes || 0) + (dbResult.sizeBytes || 0); + const manifest = { + id: backupId, + timestamp: isoString, + dateKey, + monthKey, + dbProvider: env.dbProvider, + manual, + files: { + vfs: { + name: 'vfs.tar.gz', + sizeBytes: vfsResult.sizeBytes, + sourceExists: vfsResult.exists, + }, + storage: { + name: 'storage.tar', + sizeBytes: storageResult.sizeBytes, + sourceExists: storageResult.exists, + }, + database: { + name: dbResult.success ? dbFileName : null, + sizeBytes: dbResult.sizeBytes, + backedUp: dbResult.success, + reason: dbResult.reason || null, + }, + }, + totalSizeBytes: totalSize, + durationMs: Date.now() - start, + }; + + fs.writeFileSync(path.join(stagingDir, 'manifest.json'), JSON.stringify(manifest, null, 2), 'utf8'); + + // 5. Atomically move staging to hourly directory + if (fs.existsSync(finalHourlyDir)) { + fs.rmSync(finalHourlyDir, { recursive: true, force: true }); + } + fs.renameSync(stagingDir, finalHourlyDir); + + // 6. Promote to daily backup (keep first backup of the day) + const dailyBackupName = `backup-${dateKey}`; + const targetDailyDir = path.join(paths.dailyDir, dailyBackupName); + let promotedDaily = false; + if (!fs.existsSync(targetDailyDir)) { + mirrorBackupDirectory(finalHourlyDir, targetDailyDir); + promotedDaily = true; + backupLog.info(`Promoted backup to daily tier (${dailyBackupName})`); + } + + // 7. Promote to monthly backup (keep first backup of the month forever) + const monthlyBackupName = `backup-${monthKey}`; + const targetMonthlyDir = path.join(paths.monthlyDir, monthlyBackupName); + let promotedMonthly = false; + if (!fs.existsSync(targetMonthlyDir)) { + mirrorBackupDirectory(finalHourlyDir, targetMonthlyDir); + promotedMonthly = true; + backupLog.info(`Promoted backup to monthly tier (${monthlyBackupName})`); + } + + // 8. Enforce retention limits + const pruneResult = pruneBackups(); + + const elapsed = Date.now() - start; + backupLog.info( + `Backup completed successfully in ${elapsed}ms (${(totalSize / 1024 / 1024).toFixed(2)} MB total)` + ); + + return { + success: true, + id: backupId, + timestamp: isoString, + promotedDaily, + promotedMonthly, + totalSizeBytes: totalSize, + pruneResult, + manifest, + }; + } catch (err) { + backupLog.error(`Backup failed: ${err.message}`); + if (fs.existsSync(stagingDir)) { + fs.rmSync(stagingDir, { recursive: true, force: true }); + } + throw err; + } + })(); + + try { + return await activeBackupPromise; + } finally { + activeBackupPromise = null; + } +} +// #endregion + +// #region Backup Inspection & Status +// MARK: returns list of all backups across all tiers with sizes and timestamps +export function listBackups() { + const paths = ensureBackupDirectories(); + + function readTier(dir) { + if (!fs.existsSync(dir)) return []; + return fs.readdirSync(dir, { withFileTypes: true }) + .filter((e) => e.isDirectory() && e.name.startsWith('backup-')) + .map((e) => { + const folderPath = path.join(dir, e.name); + const manifestPath = path.join(folderPath, 'manifest.json'); + let manifest = null; + if (fs.existsSync(manifestPath)) { + try { + manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + } catch { + // ignore corrupt manifest + } + } + + let totalSize = 0; + for (const file of fs.readdirSync(folderPath)) { + try { + totalSize += fs.statSync(path.join(folderPath, file)).size; + } catch { + // ignore stat errors + } + } + + return { + name: e.name, + path: folderPath, + totalSizeBytes: totalSize, + timestamp: manifest?.timestamp || null, + manifest, + }; + }) + .sort((a, b) => b.name.localeCompare(a.name)); + } + + const hourly = readTier(paths.hourlyDir); + const daily = readTier(paths.dailyDir); + const monthly = readTier(paths.monthlyDir); + + return { + counts: { + hourly: hourly.length, + daily: daily.length, + monthly: monthly.length, + }, + hourly, + daily, + monthly, + }; +} + +// MARK: returns service runtime status and config +export function getBackupServiceStatus() { + const paths = resolveBackupPaths(); + const listing = listBackups(); + return { + enabled: env.backup?.enabled ?? true, + isRunning: isServiceRunning, + dbProvider: env.dbProvider, + mssqlWarning: env.dbProvider === 'mssql', + backupDir: paths.rootDir, + retention: { + hourlyMax: env.backup?.hourlyRetentionCount || 50, + dailyMax: env.backup?.dailyRetentionCount || 50, + monthly: 'unlimited (forever)', + }, + counts: listing.counts, + latestHourly: listing.hourly[0] || null, + latestDaily: listing.daily[0] || null, + latestMonthly: listing.monthly[0] || null, + }; +} +// #endregion + +// #region Service Lifecycle +// MARK: starts the periodic backup service timer and initial check +export function startBackupService() { + if (isServiceRunning) return; + if (env.backup?.enabled === false) { + backupLog.info('Backup service is disabled by configuration.'); + return; + } + + isServiceRunning = true; + ensureBackupDirectories(); + + if (env.dbProvider === 'mssql') { + const warning = 'MSSQL database backup is not implemented; skipping database backup.'; + console.warn(`[backup] WARNING: ${warning}`); + backupLog.warn(warning); + } + + // Determine when the last hourly backup was taken + const listing = listBackups(); + const lastHourly = listing.hourly[0]; + let shouldRunInitial = false; + + if (!lastHourly || !lastHourly.timestamp) { + shouldRunInitial = true; + } else { + const lastTime = new Date(lastHourly.timestamp).getTime(); + const diffMinutes = (Date.now() - lastTime) / (1000 * 60); + if (diffMinutes >= 55) { + shouldRunInitial = true; + } + } + + if (shouldRunInitial) { + // Run after 5s initial warmup delay + scheduledTimer = setTimeout(() => { + scheduledTimer = null; + createHourlyBackup().catch((err) => { + backupLog.error(`Initial backup error: ${err.message}`); + }); + }, 5000); + } + + // Schedule next run at the start of the next hour + const now = new Date(); + const nextHour = new Date(now); + nextHour.setHours(nextHour.getHours() + 1, 0, 0, 0); + const msUntilNextHour = Math.max(1000, nextHour.getTime() - now.getTime()); + + scheduledTimer = setTimeout(() => { + scheduledTimer = null; + createHourlyBackup().catch((err) => { + backupLog.error(`Scheduled hourly backup error: ${err.message}`); + }); + + // Set recurring 1-hour interval + const intervalMs = env.backup?.intervalMs || 60 * 60 * 1000; + scheduledInterval = setInterval(() => { + createHourlyBackup().catch((err) => { + backupLog.error(`Recurring hourly backup error: ${err.message}`); + }); + }, intervalMs); + }, msUntilNextHour); + + backupLog.info(`Backup service started. Next run scheduled in ${Math.round(msUntilNextHour / 1000)}s.`); +} + +// MARK: stops the backup service and clears all active timers +export async function stopBackupService() { + isServiceRunning = false; + if (scheduledTimer) { + clearTimeout(scheduledTimer); + scheduledTimer = null; + } + if (scheduledInterval) { + clearInterval(scheduledInterval); + scheduledInterval = null; + } + + if (activeBackupPromise) { + backupLog.info('Waiting for active backup to complete...'); + await activeBackupPromise; + } + + backupLog.info('Backup service stopped.'); +} +// #endregion diff --git a/src/util/logger.js b/src/util/logger.js index 285373e..ce46d5c 100644 --- a/src/util/logger.js +++ b/src/util/logger.js @@ -66,6 +66,7 @@ const DIVISION_COLORS = { cron: chalk.hex('#eab308'), // Yellow vfs: chalk.hex('#14b8a6'), // Teal event: chalk.hex('#a855f7'), // Violet + backup: chalk.hex('#06b6d4'), // Cyan }; /**