seb/jtlPosSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

u

Browse Source
This commit is contained in:
seb
2026-07-15 03:11:30 +02:00
parent 056c9e18cd
commit 1de4144ae2
23 changed files with 921 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
scripts/s3-backup/README.md Normal file
View File

@@ -0,0 +1,51 @@
# MSSQL backup via local S3 endpoint
Custom S3-compatible HTTPS server (no MinIO). SQL Server 2022+ backs up with `BACKUP TO URL`; files land on disk under `data/sqlbackups/`.
## Quick start
```bash
npm run backup:s3
```
First run installs the CA into the `mssql` Docker container (`/var/opt/mssql/security/ca-certificates/`) and restarts SQL Server — required on Linux.
## Commands
| Command | Description |
|---------|-------------|
| `npm run backup:s3` | Start S3 endpoint + backup `MSSQL_DATABASE` from `.env` |
| `npm run backup:s3:quick` | Same, but skip PAL CA install/restart (after first setup) |
| `node scripts/s3-backup/backup.mjs --all` | Backup `eazybusiness` and `Mandant_3` |
| `node scripts/s3-backup/backup.mjs --server-only` | Run endpoint only |
| `npm run backup:s3 -- --skip-trust` | Skip CA install (npm needs `--` before script args) |
## Layout
```
scripts/s3-backup/
server.mjs S3-compatible HTTPS server (SigV4, multipart upload)
backup.mjs Orchestrator: trust CA → start server → sqlcmd BACKUP
config.mjs Host, port, credentials
ensure-certs.mjs TLS certs + Docker MSSQL PAL trust
sigv4.mjs AWS Signature V4 verification
data/sqlbackups/ Backup files written here
```
## Configuration
Reads `MSSQL_*` from repo `.env`. Optional overrides:
| Variable | Default | Description |
|----------|---------|-------------|
| `S3_BACKUP_HOST` | Docker bridge IP (`172.17.0.1`) | Host SQL Server uses in `s3://` URL |
| `S3_BACKUP_PORT` | `19443` | HTTPS port |
| `S3_BACKUP_ACCESS_KEY` | `jtlsrv-s3` | S3 access key |
| `S3_BACKUP_SECRET_KEY` | `jtlsrv-s3-secret` | S3 secret key |
| `MSSQL_DOCKER_CONTAINER` | `mssql` | Container name for CA install |
## Notes
- MSSQL runs in Docker: the endpoint binds `0.0.0.0` and uses the docker bridge IP so the container can reach it.
- SQL Server on Linux uses **SQLPAL** for outbound TLS — the CA must be in `/var/opt/mssql/security/ca-certificates/`, not only the OS trust store.
- Regenerating certs requires re-running without `--skip-trust` so PAL stays in sync.

223
scripts/s3-backup/backup.mjs
View File

@@ -1,5 +1,5 @@
#!/usr/bin/env node
import { spawnSync } from 'node:child_process';
import { fork, spawn, spawnSync } from 'node:child_process';
import fs from 'node:fs';
import https from 'node:https';
import path from 'node:path';
@@ -14,10 +14,10 @@ import {
HOST,
PORT,
SECRET_KEY,
SERVER_CERT,
s3BaseUrl,
} from './config.mjs';
import { ensureCerts, installCaTrust } from './ensure-certs.mjs';
import { startServer } from './server.mjs';
import { caTrustStatus, ensureCerts, installCaTrust } from './ensure-certs.mjs';
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
dotenv.config({ path: path.join(root, '.env') });
@@ -35,60 +35,114 @@ function sqlcmd(query) {
const port = process.env.MSSQL_PORT || '1433';
const user = process.env.MSSQL_USER || 'sa';
const password = process.env.MSSQL_PASSWORD || '';
const result = spawnSync(
'sqlcmd',
['-W', '-C', '-S', `${server},${port}`, '-U', user, '-P', password, '-Q', query],
{ encoding: 'utf8' }
);
const output = `${result.stdout || ''}${result.stderr || ''}`;
if (result.status !== 0 || /^\s*Msg \d+,/m.test(output)) {
throw new Error(output.trim() || 'sqlcmd failed');
}
return result.stdout;
return new Promise((resolve, reject) => {
const child = spawn(
'sqlcmd',
['-W', '-C', '-S', `${server},${port}`, '-U', user, '-P', password, '-Q', query],
{ encoding: 'utf8' }
);
let stdout = '';
let stderr = '';
child.stdout?.on('data', (chunk) => {
stdout += chunk;
process.stdout.write(chunk);
});
child.stderr?.on('data', (chunk) => {
stderr += chunk;
process.stderr.write(chunk);
});
child.on('close', (code) => {
const output = `${stdout}${stderr}`;
if (code !== 0 || /^\s*Msg \d+,/m.test(output)) {
reject(new Error(output.trim() || 'sqlcmd failed'));
} else {
resolve(stdout);
}
});
child.on('error', reject);
});
}
function timestamp() {
return new Date().toISOString().replace(/[:.]/g, '-');
}
async function isServerUp() {
const probeHost = /^\d+\./.test(HOST) ? '127.0.0.1' : HOST;
return new Promise((resolve) => {
const opts = {
host: probeHost,
port: PORT,
path: '/',
method: 'GET',
rejectUnauthorized: false,
};
if (!/^\d+\./.test(HOST)) {
opts.servername = HOST;
}
const req = https.request(opts, (res) => {
res.resume();
resolve(res.statusCode === 403 || res.statusCode === 200);
});
req.on('error', () => resolve(false));
req.setTimeout(1000, () => {
req.destroy();
resolve(false);
});
req.end();
});
}
async function waitForServer() {
for (let i = 0; i < 40; i++) {
const ok = await new Promise((resolve) => {
const req = https.request(
{ host: HOST, port: PORT, path: '/', method: 'GET', rejectUnauthorized: false },
(res) => {
res.resume();
resolve(res.statusCode === 403 || res.statusCode === 200);
}
);
req.on('error', () => resolve(false));
req.end();
});
if (ok) return;
if (await isServerUp()) return;
await new Promise((r) => setTimeout(r, 250));
}
throw new Error(`S3 endpoint did not start on https://${HOST}:${PORT}`);
}
function pidOnPort(port) {
const result = spawnSync('ss', ['-tlnp'], { encoding: 'utf8' });
const match = result.stdout?.match(new RegExp(`:${port}\\s+.*?pid=(\\d+)`));
return match ? Number(match[1]) : null;
}
async function stopPortListener(port) {
const stale = pidOnPort(port);
if (!stale) return;
try {
process.kill(stale);
} catch {
spawnSync('fuser', ['-k', `${port}/tcp`], { stdio: 'pipe' });
}
await new Promise((r) => setTimeout(r, 200));
}
async function ensureServerProcess() {
await stopPortListener(PORT);
return startServerProcess();
}
function ensureCredential() {
const cred = s3BaseUrl();
sqlcmd(`
IF EXISTS (SELECT 1 FROM sys.credentials WHERE name = N'${cred}')
DROP CREDENTIAL [${cred}];
CREATE CREDENTIAL [${cred}]
WITH IDENTITY = 'S3 Access Key',
SECRET = '${ACCESS_KEY}:${SECRET_KEY}';
return sqlcmd(`
IF NOT EXISTS (SELECT 1 FROM sys.credentials WHERE name = N'${cred}')
BEGIN
CREATE CREDENTIAL [${cred}]
WITH IDENTITY = 'S3 Access Key',
SECRET = '${ACCESS_KEY}:${SECRET_KEY}';
END
`);
}
function backupDatabase(db) {
async function backupDatabase(db) {
const file = `${db}-${timestamp()}.bak`;
const url = `${s3BaseUrl()}/${file}`;
console.log(`Backing up ${db} -> ${url}`);
sqlcmd(`
await sqlcmd(`
BACKUP DATABASE [${db}]
TO URL = '${url}'
WITH FORMAT, COMPRESSION, STATS = 10;
WITH FORMAT, COMPRESSION, MAXTRANSFERSIZE = 20971520, STATS = 10;
`);
const onDisk = path.join(DATA_DIR, file);
if (!fs.existsSync(onDisk)) {
@@ -99,36 +153,95 @@ WITH FORMAT, COMPRESSION, STATS = 10;
return onDisk;
}
function waitForSql() {
for (let i = 0; i < 60; i++) {
const result = spawnSync(
'sqlcmd',
[
'-W',
'-C',
'-S',
`${process.env.MSSQL_SERVER || 'localhost'},${process.env.MSSQL_PORT || '1433'}`,
'-U',
process.env.MSSQL_USER || 'sa',
'-P',
process.env.MSSQL_PASSWORD || '',
'-Q',
'SELECT 1',
],
{ encoding: 'utf8' }
);
if (result.status === 0 && !/Msg \d+,/.test(result.stdout || '')) {
return;
}
spawnSync('sleep', ['2']);
}
throw new Error('MSSQL did not become ready');
}
function startServerProcess() {
const child = fork(new URL('./server.mjs', import.meta.url), {
env: { ...process.env, S3_BACKUP_CHILD: '1' },
stdio: 'inherit',
});
return child;
}
async function main() {
ensureCerts(HOST);
if (!skipTrust) {
const trusted = installCaTrust();
if (!trusted) {
if (skipTrust) {
const status = caTrustStatus();
if (!status.inSync) {
throw new Error(
'PAL CA is out of sync with scripts/s3-backup/certs/ca.pem. Run: npm run backup:s3'
);
}
console.log('Skipping PAL CA install (--skip-trust)');
if (!fs.existsSync(SERVER_CERT)) {
throw new Error('No TLS certs found. Run: npm run backup:s3');
}
} else {
ensureCerts(HOST);
const trust = installCaTrust();
if (!trust.ok) {
console.warn('Could not install CA into MSSQL container. Run:');
console.warn(` docker cp scripts/s3-backup/certs/ca.pem mssql:/tmp/jtlsrv-s3-ca.pem`);
console.warn(' docker exec mssql bash -lc "cp /tmp/jtlsrv-s3-ca.pem /usr/local/share/ca-certificates/jtlsrv-s3.crt && update-ca-certificates"');
console.warn(' docker exec -u root mssql mkdir -p /var/opt/mssql/security/ca-certificates');
console.warn(' docker cp scripts/s3-backup/certs/ca.pem mssql:/var/opt/mssql/security/ca-certificates/jtlsrv-s3-ca.pem');
console.warn(' docker exec -u root mssql chown mssql:mssql /var/opt/mssql/security/ca-certificates/jtlsrv-s3-ca.pem');
console.warn(' docker restart mssql');
} else {
console.log('Installed S3 CA into MSSQL container');
console.log('Installed S3 CA into MSSQL PAL trust store');
if (trust.restarted) {
console.log('Waiting for MSSQL to restart...');
waitForSql();
}
}
}
await startServer();
await waitForServer();
const serverChild = await ensureServerProcess();
try {
await waitForServer();
if (serverOnly) {
console.log('Server running (--server-only). Ctrl+C to stop.');
return;
}
if (serverOnly) {
console.log('Server running (--server-only). Ctrl+C to stop.');
await new Promise((resolve) => serverChild.on('exit', resolve));
return;
}
ensureCredential();
const saved = [];
for (const db of databases) {
saved.push(backupDatabase(db));
}
await ensureCredential();
const saved = [];
for (const db of databases) {
saved.push(await backupDatabase(db));
}
console.log('\nBackups on disk:');
for (const file of saved) {
console.log(` ${file}`);
console.log('\nBackups on disk:');
for (const file of saved) {
console.log(` ${file}`);
}
} finally {
if (!serverOnly) {
serverChild.kill();
await stopPortListener(PORT);
}
}
}

106
scripts/s3-backup/ensure-certs.mjs
View File

@@ -5,16 +5,18 @@ import path from 'node:path';
import { CA_CERT, CERTS_DIR, HOST, SERVER_CERT, SERVER_KEY } from './config.mjs';
const SQL_CA_DIR = '/var/opt/mssql/security/ca-certificates';
export function certHosts(host) {
const hosts = new Set([host, '127.0.0.1', 'localhost']);
const hosts = new Set([host, '127.0.0.1', 'localhost', 's3backup.local']);
for (const iface of Object.values(os.networkInterfaces())) {
for (const addr of iface || []) {
if (addr.family === 'IPv4' && !addr.internal) {
if (addr.family === 'IPv4' && !addr.internal && !addr.address.startsWith('169.254.')) {
hosts.add(addr.address);
}
}
}
return [...hosts];
return [...hosts].sort();
}
export function ensureCerts(host = HOST) {
@@ -81,18 +83,99 @@ ${altNames}
fs.writeFileSync(marker, hostList);
}
export function caTrustStatus(container = process.env.MSSQL_DOCKER_CONTAINER || 'mssql') {
if (!fs.existsSync(CA_CERT)) {
return { ok: false, inSync: false, restarted: false };
}
const localFp = execSync(`openssl x509 -in "${CA_CERT}" -noout -fingerprint -sha256`, {
encoding: 'utf8',
}).trim();
const remoteFp = spawnSync(
'docker',
[
'exec',
container,
'bash',
'-lc',
`test -f ${SQL_CA_DIR}/jtlsrv-s3-ca.pem && openssl x509 -in ${SQL_CA_DIR}/jtlsrv-s3-ca.pem -noout -fingerprint -sha256`,
],
{ encoding: 'utf8' }
);
const inSync = remoteFp.status === 0 && remoteFp.stdout.trim() === localFp;
return { ok: remoteFp.status === 0, inSync, restarted: false };
}
export function installCaTrust(container = process.env.MSSQL_DOCKER_CONTAINER || 'mssql') {
if (!fs.existsSync(CA_CERT)) {
ensureCerts();
}
const copied = spawnSync('docker', ['cp', CA_CERT, `${container}:/tmp/jtlsrv-s3-ca.pem`], {
const localFp = execSync(`openssl x509 -in "${CA_CERT}" -noout -fingerprint -sha256`, {
encoding: 'utf8',
}).trim();
const remoteFp = spawnSync(
'docker',
[
'exec',
container,
'bash',
'-lc',
`test -f ${SQL_CA_DIR}/jtlsrv-s3-ca.pem && openssl x509 -in ${SQL_CA_DIR}/jtlsrv-s3-ca.pem -noout -fingerprint -sha256`,
],
{ encoding: 'utf8' }
);
if (remoteFp.status === 0 && remoteFp.stdout.trim() === localFp) {
spawnSync('docker', [
'exec',
'-u',
'root',
container,
'bash',
'-lc',
'grep -q s3backup.local /etc/hosts || echo "172.17.0.1 s3backup.local" >> /etc/hosts',
]);
return { ok: true, restarted: false };
}
spawnSync('docker', ['exec', '-u', 'root', container, 'mkdir', '-p', SQL_CA_DIR], {
stdio: 'pipe',
});
if (copied.status !== 0) {
return false;
spawnSync('docker', [
'exec',
'-u',
'root',
container,
'bash',
'-lc',
`rm -f ${SQL_CA_DIR}/*.pem ${SQL_CA_DIR}/*.crt`,
]);
spawnSync('docker', [
'exec',
'-u',
'root',
container,
'bash',
'-lc',
'grep -q s3backup.local /etc/hosts || echo "172.17.0.1 s3backup.local" >> /etc/hosts',
]);
try {
execSync('grep -q s3backup.local /etc/hosts || echo "172.17.0.1 s3backup.local" >> /etc/hosts', {
stdio: 'pipe',
});
} catch {
// optional on host
}
const installed = spawnSync(
const copied = spawnSync(
'docker',
['cp', CA_CERT, `${container}:${SQL_CA_DIR}/jtlsrv-s3-ca.pem`],
{ stdio: 'pipe' }
);
if (copied.status !== 0) {
return { ok: false, restarted: false };
}
const perms = spawnSync(
'docker',
[
'exec',
@@ -101,9 +184,14 @@ export function installCaTrust(container = process.env.MSSQL_DOCKER_CONTAINER ||
container,
'bash',
'-lc',
'cp /tmp/jtlsrv-s3-ca.pem /usr/local/share/ca-certificates/jtlsrv-s3.crt && update-ca-certificates',
`chown mssql:mssql ${SQL_CA_DIR}/jtlsrv-s3-ca.pem && chmod 644 ${SQL_CA_DIR}/jtlsrv-s3-ca.pem`,
],
{ stdio: 'pipe' }
);
return installed.status === 0;
if (perms.status !== 0) {
return { ok: false, restarted: false };
}
const restarted = spawnSync('docker', ['restart', container], { stdio: 'pipe' });
return { ok: true, restarted: restarted.status === 0 };
}

38
scripts/s3-backup/server.mjs
View File

@@ -2,7 +2,7 @@ import crypto from 'node:crypto';
import fs from 'node:fs';
import https from 'node:https';
import path from 'node:path';
import { URL } from 'node:url';
import { URL, pathToFileURL } from 'node:url';
import {
ACCESS_KEY,
@@ -32,6 +32,9 @@ function send(res, status, body = '', headers = {}) {
}
function readBody(req) {
if (req.method === 'GET' || req.method === 'HEAD' || req.method === 'DELETE') {
return Promise.resolve(Buffer.alloc(0));
}
return new Promise((resolve, reject) => {
const chunks = [];
req.on('data', (chunk) => chunks.push(chunk));
@@ -169,10 +172,17 @@ function abortMultipart(uploadId) {
return true;
}
function debug(...args) {
if (process.env.S3_BACKUP_DEBUG) {
console.error(...args);
}
}
async function handle(req, res) {
const body = await readBody(req);
debug(`${req.method} ${req.url} len=${body.length}`);
if (!authOk(req, body)) {
console.error(`${req.method} ${req.url} -> 403 auth failed`);
debug(`auth failed ${req.method} ${req.url}`);
return send(res, 403, xml('<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>'));
}
@@ -195,6 +205,10 @@ async function handle(req, res) {
return send(res, 200, listBucketXml(query.get('prefix') || ''));
}
if (req.method === 'HEAD' && !key) {
return send(res, 200, '');
}
if (req.method === 'GET' && key) {
const file = objectPath(key);
if (!fs.existsSync(file)) {
@@ -258,7 +272,6 @@ async function handle(req, res) {
}
export function startServer() {
ensureCerts(HOST);
fs.mkdirSync(DATA_DIR, { recursive: true });
fs.mkdirSync(TMP_DIR, { recursive: true });
@@ -272,7 +285,17 @@ export function startServer() {
}
);
return new Promise((resolve) => {
return new Promise((resolve, reject) => {
server.on('error', (err) => {
if (err.code === 'EADDRINUSE') {
reject(new Error(`Port ${PORT} is already in use`));
return;
}
reject(err);
});
server.on('secureConnection', (tlsSocket) => {
debug(`tls ${tlsSocket.remoteAddress}:${tlsSocket.remotePort}`);
});
server.listen(PORT, BIND, () => {
console.log(`S3 endpoint https://${HOST}:${PORT}/${BUCKET} -> ${DATA_DIR}`);
resolve(server);
@@ -280,6 +303,9 @@ export function startServer() {
});
}
if (import.meta.url === `file://${process.argv[1]}`) {
startServer();
if (process.env.S3_BACKUP_CHILD || import.meta.url === pathToFileURL(process.argv[1]).href) {
startServer().catch((err) => {
console.error(err.message || err);
process.exit(1);
});
}

12
scripts/s3-backup/sigv4.mjs
View File

@@ -22,7 +22,6 @@ function parseAuthHeader(header) {
accessKey: credential[0],
date: credential[1],
region: credential[2],
service: credential[3],
signedHeaders: parts.SignedHeaders.split(';'),
signature: parts.Signature,
};
@@ -59,6 +58,7 @@ export function verifyRequest(req, body, { accessKey, secretKey, region = 'us-ea
return false;
}
const regionToUse = parsed.region || region;
const amzDate = getHeader(req, 'x-amz-date');
const declaredPayload = getHeader(req, 'x-amz-content-sha256');
const payloadHash =
@@ -72,16 +72,16 @@ export function verifyRequest(req, body, { accessKey, secretKey, region = 'us-ea
payloadHash,
].join('\n');
const scope = `${parsed.date}/${region}/s3/aws4_request`;
const scope = `${parsed.date}/${regionToUse}/s3/aws4_request`;
const stringToSign = ['AWS4-HMAC-SHA256', amzDate, scope, hash(canonical)].join('\n');
const signingKey = hmac(
hmac(
hmac(hmac(`AWS4${secretKey}`, parsed.date), region),
's3'
),
hmac(hmac(hmac(`AWS4${secretKey}`, parsed.date), regionToUse), 's3'),
'aws4_request'
);
const expected = hmac(signingKey, stringToSign, 'hex');
if (expected.length !== parsed.signature.length) {
return false;
}
return crypto.timingSafeEqual(Buffer.from(expected, 'hex'), Buffer.from(parsed.signature, 'hex'));
}