u
This commit is contained in:
seb
285
scripts/s3-backup/server.mjs
Normal file
285
scripts/s3-backup/server.mjs
Normal file
@@ -0,0 +1,285 @@
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import https from 'node:https';
|
||||
import path from 'node:path';
|
||||
import { URL } from 'node:url';
|
||||
|
||||
import {
|
||||
ACCESS_KEY,
|
||||
BIND,
|
||||
BUCKET,
|
||||
DATA_DIR,
|
||||
HOST,
|
||||
PORT,
|
||||
REGION,
|
||||
SECRET_KEY,
|
||||
SERVER_CERT,
|
||||
SERVER_KEY,
|
||||
TMP_DIR,
|
||||
} from './config.mjs';
|
||||
import { ensureCerts } from './ensure-certs.mjs';
|
||||
import { etagFor, verifyRequest } from './sigv4.mjs';
|
||||
|
||||
const uploads = new Map();
|
||||
|
||||
function xml(body) {
|
||||
return `<?xml version="1.0" encoding="UTF-8"?>\n${body}`;
|
||||
}
|
||||
|
||||
function send(res, status, body = '', headers = {}) {
|
||||
res.writeHead(status, { 'Content-Type': 'application/xml', ...headers });
|
||||
res.end(body);
|
||||
}
|
||||
|
||||
function readBody(req) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks = [];
|
||||
req.on('data', (chunk) => chunks.push(chunk));
|
||||
req.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function objectPath(key) {
|
||||
return path.join(DATA_DIR, key);
|
||||
}
|
||||
|
||||
function ensureObjectDir(key) {
|
||||
fs.mkdirSync(path.dirname(objectPath(key)), { recursive: true });
|
||||
}
|
||||
|
||||
function listObjects(prefix = '') {
|
||||
if (!fs.existsSync(DATA_DIR)) return [];
|
||||
const out = [];
|
||||
const walk = (dir, rel = '') => {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const relPath = rel ? `${rel}/${entry.name}` : entry.name;
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(full, relPath);
|
||||
} else if (!prefix || relPath.startsWith(prefix)) {
|
||||
const stat = fs.statSync(full);
|
||||
out.push({ key: relPath, size: stat.size, mtime: stat.mtime });
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(DATA_DIR);
|
||||
return out;
|
||||
}
|
||||
|
||||
function parseRoute(urlString) {
|
||||
const url = new URL(urlString, 'https://local');
|
||||
const parts = url.pathname.split('/').filter(Boolean);
|
||||
if (parts.length === 0) {
|
||||
return { type: 'root' };
|
||||
}
|
||||
if (parts[0] !== BUCKET) {
|
||||
return { type: 'missing' };
|
||||
}
|
||||
const key = parts.slice(1).join('/');
|
||||
return { type: 'object', key, query: url.searchParams };
|
||||
}
|
||||
|
||||
function authOk(req, body) {
|
||||
if (!req.headers.authorization) return false;
|
||||
return verifyRequest(req, body, {
|
||||
accessKey: ACCESS_KEY,
|
||||
secretKey: SECRET_KEY,
|
||||
region: REGION,
|
||||
});
|
||||
}
|
||||
|
||||
function listBucketsXml() {
|
||||
return xml(`<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Owner><ID>jtlsrv</ID><DisplayName>jtlsrv</DisplayName></Owner>
|
||||
<Buckets>
|
||||
<Bucket>
|
||||
<Name>${BUCKET}</Name>
|
||||
<CreationDate>2026-01-01T00:00:00.000Z</CreationDate>
|
||||
</Bucket>
|
||||
</Buckets>
|
||||
</ListAllMyBucketsResult>`);
|
||||
}
|
||||
|
||||
function listBucketXml(prefix) {
|
||||
const items = listObjects(prefix).map((item) => {
|
||||
const etag = etagFor(fs.readFileSync(path.join(DATA_DIR, item.key)));
|
||||
return `<Contents>
|
||||
<Key>${item.key}</Key>
|
||||
<LastModified>${item.mtime.toISOString()}</LastModified>
|
||||
<ETag>${etag}</ETag>
|
||||
<Size>${item.size}</Size>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
</Contents>`;
|
||||
}).join('\n');
|
||||
return xml(`<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>${BUCKET}</Name>
|
||||
<Prefix>${prefix}</Prefix>
|
||||
<MaxKeys>1000</MaxKeys>
|
||||
<IsTruncated>false</IsTruncated>
|
||||
${items}
|
||||
</ListBucketResult>`);
|
||||
}
|
||||
|
||||
function initiateMultipart(key) {
|
||||
const uploadId = crypto.randomUUID();
|
||||
const dir = path.join(TMP_DIR, uploadId);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
uploads.set(uploadId, { key, dir, parts: new Map() });
|
||||
return xml(`<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Bucket>${BUCKET}</Bucket>
|
||||
<Key>${key}</Key>
|
||||
<UploadId>${uploadId}</UploadId>
|
||||
</InitiateMultipartUploadResult>`);
|
||||
}
|
||||
|
||||
function savePart(uploadId, partNumber, body) {
|
||||
const upload = uploads.get(uploadId);
|
||||
if (!upload) return null;
|
||||
const partPath = path.join(upload.dir, String(partNumber).padStart(5, '0'));
|
||||
fs.writeFileSync(partPath, body);
|
||||
upload.parts.set(partNumber, partPath);
|
||||
return etagFor(body);
|
||||
}
|
||||
|
||||
function completeMultipart(uploadId) {
|
||||
const upload = uploads.get(uploadId);
|
||||
if (!upload) return null;
|
||||
ensureObjectDir(upload.key);
|
||||
const partNumbers = [...upload.parts.keys()].sort((a, b) => a - b);
|
||||
const chunks = partNumbers.map((n) => fs.readFileSync(upload.parts.get(n)));
|
||||
const finalData = Buffer.concat(chunks);
|
||||
fs.writeFileSync(objectPath(upload.key), finalData);
|
||||
fs.rmSync(upload.dir, { recursive: true, force: true });
|
||||
uploads.delete(uploadId);
|
||||
const etag = etagFor(finalData);
|
||||
return xml(`<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Location>https://${HOST}:${PORT}/${BUCKET}/${upload.key}</Location>
|
||||
<Bucket>${BUCKET}</Bucket>
|
||||
<Key>${upload.key}</Key>
|
||||
<ETag>${etag}</ETag>
|
||||
</CompleteMultipartUploadResult>`);
|
||||
}
|
||||
|
||||
function abortMultipart(uploadId) {
|
||||
const upload = uploads.get(uploadId);
|
||||
if (!upload) return false;
|
||||
fs.rmSync(upload.dir, { recursive: true, force: true });
|
||||
uploads.delete(uploadId);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function handle(req, res) {
|
||||
const body = await readBody(req);
|
||||
if (!authOk(req, body)) {
|
||||
console.error(`${req.method} ${req.url} -> 403 auth failed`);
|
||||
return send(res, 403, xml('<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>'));
|
||||
}
|
||||
|
||||
const route = parseRoute(req.url);
|
||||
if (route.type === 'missing') {
|
||||
return send(res, 404, xml('<Error><Code>NoSuchBucket</Code><Message>Not Found</Message></Error>'));
|
||||
}
|
||||
|
||||
if (route.type === 'root' && req.method === 'GET') {
|
||||
return send(res, 200, listBucketsXml());
|
||||
}
|
||||
|
||||
if (route.type !== 'object') {
|
||||
return send(res, 404, xml('<Error><Code>NoSuchKey</Code><Message>Not Found</Message></Error>'));
|
||||
}
|
||||
|
||||
const { key, query } = route;
|
||||
|
||||
if (req.method === 'GET' && !key) {
|
||||
return send(res, 200, listBucketXml(query.get('prefix') || ''));
|
||||
}
|
||||
|
||||
if (req.method === 'GET' && key) {
|
||||
const file = objectPath(key);
|
||||
if (!fs.existsSync(file)) {
|
||||
return send(res, 404, xml('<Error><Code>NoSuchKey</Code><Message>Not Found</Message></Error>'));
|
||||
}
|
||||
const data = fs.readFileSync(file);
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'Content-Length': data.length,
|
||||
ETag: etagFor(data),
|
||||
});
|
||||
return res.end(data);
|
||||
}
|
||||
|
||||
if (req.method === 'HEAD' && key) {
|
||||
const file = objectPath(key);
|
||||
if (!fs.existsSync(file)) {
|
||||
return send(res, 404, '');
|
||||
}
|
||||
const stat = fs.statSync(file);
|
||||
return send(res, 200, '', {
|
||||
'Content-Length': stat.size,
|
||||
ETag: etagFor(fs.readFileSync(file)),
|
||||
});
|
||||
}
|
||||
|
||||
if (req.method === 'PUT' && key && query.has('uploadId') && query.has('partNumber')) {
|
||||
const etag = savePart(query.get('uploadId'), Number(query.get('partNumber')), body);
|
||||
if (!etag) {
|
||||
return send(res, 404, xml('<Error><Code>NoSuchUpload</Code><Message>Not Found</Message></Error>'));
|
||||
}
|
||||
return send(res, 200, '', { ETag: etag });
|
||||
}
|
||||
|
||||
if (req.method === 'PUT' && key) {
|
||||
ensureObjectDir(key);
|
||||
fs.writeFileSync(objectPath(key), body);
|
||||
return send(res, 200, '', { ETag: etagFor(body) });
|
||||
}
|
||||
|
||||
if (req.method === 'POST' && key && query.has('uploads')) {
|
||||
return send(res, 200, initiateMultipart(key));
|
||||
}
|
||||
|
||||
if (req.method === 'POST' && key && query.has('uploadId')) {
|
||||
const result = completeMultipart(query.get('uploadId'));
|
||||
if (!result) {
|
||||
return send(res, 404, xml('<Error><Code>NoSuchUpload</Code><Message>Not Found</Message></Error>'));
|
||||
}
|
||||
return send(res, 200, result);
|
||||
}
|
||||
|
||||
if (req.method === 'DELETE' && key && query.has('uploadId')) {
|
||||
if (!abortMultipart(query.get('uploadId'))) {
|
||||
return send(res, 404, xml('<Error><Code>NoSuchUpload</Code><Message>Not Found</Message></Error>'));
|
||||
}
|
||||
return send(res, 204, '');
|
||||
}
|
||||
|
||||
return send(res, 405, xml('<Error><Code>MethodNotAllowed</Code><Message>Not allowed</Message></Error>'));
|
||||
}
|
||||
|
||||
export function startServer() {
|
||||
ensureCerts(HOST);
|
||||
fs.mkdirSync(DATA_DIR, { recursive: true });
|
||||
fs.mkdirSync(TMP_DIR, { recursive: true });
|
||||
|
||||
const server = https.createServer(
|
||||
{ key: fs.readFileSync(SERVER_KEY), cert: fs.readFileSync(SERVER_CERT) },
|
||||
(req, res) => {
|
||||
handle(req, res).catch((err) => {
|
||||
console.error(err);
|
||||
send(res, 500, xml('<Error><Code>InternalError</Code><Message>Server error</Message></Error>'));
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
return new Promise((resolve) => {
|
||||
server.listen(PORT, BIND, () => {
|
||||
console.log(`S3 endpoint https://${HOST}:${PORT}/${BUCKET} -> ${DATA_DIR}`);
|
||||
resolve(server);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
startServer();
|
||||
}
|
||||
Reference in New Issue
Block a user