Compare commits
2 Commits
eed10ea38d
...
7fb7435779
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7fb7435779 | ||
|
|
67678038c0 |
@@ -684,7 +684,7 @@ _none_
|
|||||||
- Returns: `200 str`
|
- Returns: `200 str`
|
||||||
#### Execute a virtual filesystem applet — `POST /vfs/{*path}`
|
#### Execute a virtual filesystem applet — `POST /vfs/{*path}`
|
||||||
- Params: `path: str, body?: { input?: {}, timeout?: int, max_output?: int }`
|
- Params: `path: str, body?: { input?: {}, timeout?: int, max_output?: int }`
|
||||||
- Returns: `200 { ok: bool, result: any|null, stdout: str, consoleLines: int, durationMs: int, truncated?: bool, truncationNote?: str }`
|
- Returns: `200 { ok: bool, result: str, columns: { name: str, type: str }[], rows: int, stdout: str, consoleLines: int, durationMs: int, warnings?: str[], truncated?: bool, truncationNote?: str }`
|
||||||
#### Create or overwrite a virtual filesystem file — `PUT /vfs/{*path}`
|
#### Create or overwrite a virtual filesystem file — `PUT /vfs/{*path}`
|
||||||
- Params: `path: str, body: { content: str, mime?: str }`
|
- Params: `path: str, body: { content: str, mime?: str }`
|
||||||
- Returns: `200 { success: bool, file: { name: str, size: int, mime: str, updated_at: str } }`
|
- Returns: `200 { success: bool, file: { name: str, size: int, mime: str, updated_at: str } }`
|
||||||
|
|||||||
@@ -9,10 +9,17 @@ and returns the exported function's result.
|
|||||||
|
|
||||||
The applet file must begin with two metadata lines:
|
The applet file must begin with two metadata lines:
|
||||||
`// @applet-input-schema {JSON Schema}` followed by
|
`// @applet-input-schema {JSON Schema}` followed by
|
||||||
`// @applet-output-schema {JSON Schema}`. The file exports a function that receives the posted `input`
|
`// @applet-output-schema {name:type;name:type}`. The input schema describes the GUI form posted as
|
||||||
as `applet_input`, and execution runs in the same isolated worker sandbox as the `javascript_execute`
|
`input`. The output line declares the RESULT COLUMNS as typed CSV — `name:type` pairs separated by `;`
|
||||||
tool — including authenticated `fetch()` against this API, `fs`/`vfs`, `sharp`, ExcelJS, PDFKit and
|
(e.g. `id:integer;name:string;active:boolean`); allowed types are `string`, `integer`, `number`,
|
||||||
WebCrypto globals.
|
`boolean` and `date`. The applet exports a function that receives the posted `input` as `applet_input`
|
||||||
|
and returns a row object (or an array of row objects) keyed by those columns.
|
||||||
|
|
||||||
|
Applets return typed CSV, not JSON: `result` is a `;`-separated CSV string with a header row, and the
|
||||||
|
response also carries `columns` (name/type) and `rows` (data row count). Only declared columns are
|
||||||
|
emitted, in header order; missing values become empty cells. Execution runs in the same isolated worker
|
||||||
|
sandbox as the `javascript_execute` tool — including authenticated `fetch()` against this API,
|
||||||
|
`fs`/`vfs`, `sharp`, ExcelJS, PDFKit and WebCrypto globals.
|
||||||
|
|
||||||
The path is taken from the URL (like `DELETE /vfs/{*path}`); the run arguments are the JSON body.
|
The path is taken from the URL (like `DELETE /vfs/{*path}`); the run arguments are the JSON body.
|
||||||
Path traversal (`..`), null bytes, and backslashes are rejected with HTTP 400, as is a path that does
|
Path traversal (`..`), null bytes, and backslashes are rejected with HTTP 400, as is a path that does
|
||||||
@@ -52,7 +59,17 @@ Applet executed successfully
|
|||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"ok": true,
|
"ok": true,
|
||||||
"result": null,
|
"result": "id;name\n1;Kaffee",
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"name": "…",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"rows": 1,
|
||||||
|
"warnings": [
|
||||||
|
"…"
|
||||||
|
],
|
||||||
"stdout": "…",
|
"stdout": "…",
|
||||||
"consoleLines": 1,
|
"consoleLines": 1,
|
||||||
"durationMs": 1,
|
"durationMs": 1,
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import { ensureConnection } from '../db/store.js';
|
|||||||
import { closeAll as closeAllSse } from './sse.js';
|
import { closeAll as closeAllSse } from './sse.js';
|
||||||
import { closeAllManagers } from '../services/conversationManager.js';
|
import { closeAllManagers } from '../services/conversationManager.js';
|
||||||
import { shutdownImageCropWorkers } from '../services/imageCrop.js';
|
import { shutdownImageCropWorkers } from '../services/imageCrop.js';
|
||||||
|
import { shutdownJavascriptSandboxWorkers } from '../services/javascriptSandbox.js';
|
||||||
import { getOrCreateCert } from '../util/cert.js';
|
import { getOrCreateCert } from '../util/cert.js';
|
||||||
import { createLogger } from '../util/logger.js';
|
import { createLogger } from '../util/logger.js';
|
||||||
import v1Router from '../v1/index.js';
|
import v1Router from '../v1/index.js';
|
||||||
@@ -242,6 +243,8 @@ export async function stopApi() {
|
|||||||
// Pooled image-crop workers are live threads and must be terminated too, or the
|
// Pooled image-crop workers are live threads and must be terminated too, or the
|
||||||
// process stays alive after the last request has been served.
|
// process stays alive after the last request has been served.
|
||||||
await shutdownImageCropWorkers();
|
await shutdownImageCropWorkers();
|
||||||
|
// Same for the pooled sandbox threads that run `javascript_execute` / applets.
|
||||||
|
await shutdownJavascriptSandboxWorkers();
|
||||||
const closers = [];
|
const closers = [];
|
||||||
if (server) {
|
if (server) {
|
||||||
const s = server;
|
const s = server;
|
||||||
|
|||||||
@@ -72,10 +72,17 @@ export function register(app, basePath = '') {
|
|||||||
*
|
*
|
||||||
* The applet file must begin with two metadata lines:
|
* The applet file must begin with two metadata lines:
|
||||||
* `// @applet-input-schema {JSON Schema}` followed by
|
* `// @applet-input-schema {JSON Schema}` followed by
|
||||||
* `// @applet-output-schema {JSON Schema}`. The file exports a function that receives the posted `input`
|
* `// @applet-output-schema {name:type;name:type}`. The input schema describes the GUI form posted as
|
||||||
* as `applet_input`, and execution runs in the same isolated worker sandbox as the `javascript_execute`
|
* `input`. The output line declares the RESULT COLUMNS as typed CSV — `name:type` pairs separated by `;`
|
||||||
* tool — including authenticated `fetch()` against this API, `fs`/`vfs`, `sharp`, ExcelJS, PDFKit and
|
* (e.g. `id:integer;name:string;active:boolean`); allowed types are `string`, `integer`, `number`,
|
||||||
* WebCrypto globals.
|
* `boolean` and `date`. The applet exports a function that receives the posted `input` as `applet_input`
|
||||||
|
* and returns a row object (or an array of row objects) keyed by those columns.
|
||||||
|
*
|
||||||
|
* Applets return typed CSV, not JSON: `result` is a `;`-separated CSV string with a header row, and the
|
||||||
|
* response also carries `columns` (name/type) and `rows` (data row count). Only declared columns are
|
||||||
|
* emitted, in header order; missing values become empty cells. Execution runs in the same isolated worker
|
||||||
|
* sandbox as the `javascript_execute` tool — including authenticated `fetch()` against this API,
|
||||||
|
* `fs`/`vfs`, `sharp`, ExcelJS, PDFKit and WebCrypto globals.
|
||||||
*
|
*
|
||||||
* The path is taken from the URL (like `DELETE /vfs/{*path}`); the run arguments are the JSON body.
|
* The path is taken from the URL (like `DELETE /vfs/{*path}`); the run arguments are the JSON body.
|
||||||
* Path traversal (`..`), null bytes, and backslashes are rejected with HTTP 400, as is a path that does
|
* Path traversal (`..`), null bytes, and backslashes are rejected with HTTP 400, as is a path that does
|
||||||
@@ -115,13 +122,34 @@ export function register(app, basePath = '') {
|
|||||||
* application/json:
|
* application/json:
|
||||||
* schema:
|
* schema:
|
||||||
* type: object
|
* type: object
|
||||||
* required: [ok, result, stdout, consoleLines, durationMs]
|
* required: [ok, result, columns, rows, stdout, consoleLines, durationMs]
|
||||||
* properties:
|
* properties:
|
||||||
* ok:
|
* ok:
|
||||||
* type: boolean
|
* type: boolean
|
||||||
* result:
|
* result:
|
||||||
* description: The value returned by the applet's exported function
|
* type: string
|
||||||
* nullable: true
|
* description: Typed CSV table (`;`-separated, header row first) built from the returned rows and the declared output columns
|
||||||
|
* example: "id;name\n1;Kaffee"
|
||||||
|
* columns:
|
||||||
|
* type: array
|
||||||
|
* description: Declared output columns, in header order
|
||||||
|
* items:
|
||||||
|
* type: object
|
||||||
|
* required: [name, type]
|
||||||
|
* properties:
|
||||||
|
* name:
|
||||||
|
* type: string
|
||||||
|
* type:
|
||||||
|
* type: string
|
||||||
|
* enum: [string, integer, number, boolean, date]
|
||||||
|
* rows:
|
||||||
|
* type: integer
|
||||||
|
* description: Number of data rows in `result` (excludes the header row)
|
||||||
|
* warnings:
|
||||||
|
* type: array
|
||||||
|
* description: Present when cells were dropped for not matching their declared type
|
||||||
|
* items:
|
||||||
|
* type: string
|
||||||
* stdout:
|
* stdout:
|
||||||
* type: string
|
* type: string
|
||||||
* description: Captured console output
|
* description: Captured console output
|
||||||
|
|||||||
406
src/services/javascriptSandbox.js
Normal file
406
src/services/javascriptSandbox.js
Normal file
@@ -0,0 +1,406 @@
|
|||||||
|
import { Worker } from 'worker_threads';
|
||||||
|
import { fileURLToPath } from 'url';
|
||||||
|
import env from '../config/env.js';
|
||||||
|
|
||||||
|
// The sandbox worker pays a large fixed startup cost: importing vm2, sharp, exceljs, pdfkit,
|
||||||
|
// react-canvas-label-editor AND `vfs.js` (which pulls in libdashstore) costs ~700 ms in a fresh
|
||||||
|
// thread, while the snippet itself runs in ~2 ms. ESM modules are never shared between worker
|
||||||
|
// threads, so spawning one thread per applet run re-paid that cost on every request.
|
||||||
|
//
|
||||||
|
// This pool keeps threads alive and hands them one job at a time. Threads are keyed by VFS root
|
||||||
|
// (`data/vfs/<companyId>`), so a reused thread always has the right `vfsDir` baked in and two
|
||||||
|
// companies can never share a thread.
|
||||||
|
const WORKER_PATH = fileURLToPath(new URL('../tools/javascript-worker.js', import.meta.url));
|
||||||
|
const ROOT_DIR = fileURLToPath(new URL('../..', import.meta.url));
|
||||||
|
const WORKER_BOOT_TIMEOUT_MS = 15000;
|
||||||
|
|
||||||
|
// Two threads per company rides out the common "run it, then run it again" loop without queueing.
|
||||||
|
const MAX_WORKERS_PER_COMPANY = 2;
|
||||||
|
// ...but the process is shared by every company, so the total stays bounded. Each thread holds
|
||||||
|
// sharp/libvips plus a NodeVM, so 5 is a deliberate memory ceiling rather than a core count.
|
||||||
|
const MAX_WORKERS_TOTAL = 5;
|
||||||
|
|
||||||
|
const DEFAULT_BUFFER_LIMIT = 32 * 1024 * 1024;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {object} PoolEntry
|
||||||
|
* @property {Worker} worker
|
||||||
|
* @property {number} id Monotonic thread id, used to tag job ids.
|
||||||
|
* @property {string} poolKey VFS root this thread was spawned for.
|
||||||
|
* @property {boolean} busy
|
||||||
|
* @property {number} lastUsed `Date.now()` of the last job handoff (eviction order).
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** @type {Array<PoolEntry>} */
|
||||||
|
const pool = [];
|
||||||
|
/** @type {Array<{ resolve: Function, reject: Function, timer: NodeJS.Timeout, poolKey: string, job: object, timeoutMs: number }>} */
|
||||||
|
const queue = [];
|
||||||
|
let workerSeq = 0;
|
||||||
|
let jobSeq = 0;
|
||||||
|
let shuttingDown = false;
|
||||||
|
/** Serializes pool growth so concurrent callers cannot overshoot either cap. */
|
||||||
|
let growing = Promise.resolve();
|
||||||
|
|
||||||
|
function countFor(poolKey) {
|
||||||
|
return pool.filter((entry) => entry.poolKey === poolKey).length;
|
||||||
|
}
|
||||||
|
|
||||||
|
function findIdle(poolKey) {
|
||||||
|
return pool.find((entry) => !entry.busy && entry.poolKey === poolKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
function dropWorker(entry) {
|
||||||
|
const index = pool.indexOf(entry);
|
||||||
|
if (index !== -1) pool.splice(index, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reclaims an idle thread belonging to a DIFFERENT company so a queued job can be served.
|
||||||
|
*
|
||||||
|
* Without this the total cap would deadlock: with 5 threads spread over three companies, a
|
||||||
|
* fourth company's job could never be dispatched (no thread for it) and never spawn (cap hit).
|
||||||
|
* Only genuinely idle threads are evicted, and preferentially ones whose own company has
|
||||||
|
* nothing waiting.
|
||||||
|
*
|
||||||
|
* @param {string} poolKey
|
||||||
|
* @returns {boolean} whether a thread was reclaimed
|
||||||
|
*/
|
||||||
|
function evictIdleOtherCompany(poolKey) {
|
||||||
|
const candidates = pool.filter((entry) => !entry.busy && entry.poolKey !== poolKey);
|
||||||
|
if (candidates.length === 0) return false;
|
||||||
|
|
||||||
|
const idleOthers = new Set(
|
||||||
|
queue.filter((item) => item.poolKey !== poolKey).map((item) => item.poolKey)
|
||||||
|
);
|
||||||
|
candidates.sort((a, b) => {
|
||||||
|
const aWanted = idleOthers.has(a.poolKey) ? 1 : 0;
|
||||||
|
const bWanted = idleOthers.has(b.poolKey) ? 1 : 0;
|
||||||
|
return aWanted - bWanted || a.lastUsed - b.lastUsed;
|
||||||
|
});
|
||||||
|
|
||||||
|
const victim = candidates[0];
|
||||||
|
dropWorker(victim);
|
||||||
|
try { victim.worker.terminate(); } catch { /* already closed */ }
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Spawns a thread for `vfsRoot` and resolves once it has reported `{ ready: true }`.
|
||||||
|
*
|
||||||
|
* The worker signals readiness only after its module body (and therefore every import) has
|
||||||
|
* finished, so a thread is never handed a job while it is still booting — which is what makes
|
||||||
|
* the reuse actually cheap.
|
||||||
|
*
|
||||||
|
* @param {string} vfsRoot VFS root of the company this thread serves (the pool key).
|
||||||
|
* @param {number} googleAuthId
|
||||||
|
* @returns {Promise<PoolEntry>}
|
||||||
|
*/
|
||||||
|
function spawnWorker(vfsRoot, googleAuthId) {
|
||||||
|
const entry = { worker: null, id: ++workerSeq, poolKey: vfsRoot, busy: false, lastUsed: Date.now() };
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const bootTimer = setTimeout(() => {
|
||||||
|
cleanup();
|
||||||
|
try { entry.worker?.terminate(); } catch { /* already closed */ }
|
||||||
|
reject(new Error(`Sandbox worker failed to start within ${WORKER_BOOT_TIMEOUT_MS} ms`));
|
||||||
|
}, WORKER_BOOT_TIMEOUT_MS);
|
||||||
|
|
||||||
|
function cleanup() {
|
||||||
|
clearTimeout(bootTimer);
|
||||||
|
entry.worker?.off('message', onBooted);
|
||||||
|
entry.worker?.off('error', onError);
|
||||||
|
entry.worker?.off('exit', onExit);
|
||||||
|
}
|
||||||
|
|
||||||
|
function onBooted(msg) {
|
||||||
|
if (!msg || msg.ready !== true) return;
|
||||||
|
cleanup();
|
||||||
|
resolve(entry);
|
||||||
|
}
|
||||||
|
|
||||||
|
function onError(err) {
|
||||||
|
cleanup();
|
||||||
|
reject(err instanceof Error ? err : new Error(err?.message || 'Sandbox worker failed to start'));
|
||||||
|
}
|
||||||
|
|
||||||
|
function onExit(code) {
|
||||||
|
cleanup();
|
||||||
|
reject(new Error(`Sandbox worker exited before becoming ready (code ${code})`));
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
entry.worker = new Worker(WORKER_PATH, {
|
||||||
|
workerData: {
|
||||||
|
root: ROOT_DIR,
|
||||||
|
bufferLimit: DEFAULT_BUFFER_LIMIT,
|
||||||
|
dashbackUrl: `http://127.0.0.1:${env.port}/api`,
|
||||||
|
vfsDir: vfsRoot,
|
||||||
|
googleAuthId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
cleanup();
|
||||||
|
reject(err);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
entry.worker.on('message', onBooted);
|
||||||
|
entry.worker.on('error', onError);
|
||||||
|
entry.worker.on('exit', onExit);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns an idle thread for `poolKey`, spawning (or evicting) to grow the pool when allowed.
|
||||||
|
* Resolves to `null` when every permitted thread is busy — the caller then queues.
|
||||||
|
*
|
||||||
|
* @param {string} poolKey
|
||||||
|
* @param {number} googleAuthId
|
||||||
|
* @returns {Promise<PoolEntry | null>}
|
||||||
|
*/
|
||||||
|
async function acquireWorker(poolKey, googleAuthId) {
|
||||||
|
const idle = findIdle(poolKey);
|
||||||
|
// Reserve SYNCHRONOUSLY, before any `await`. Otherwise two concurrent callers both pass
|
||||||
|
// this check while the thread is still unmarked, and both post to the same worker.
|
||||||
|
if (idle) {
|
||||||
|
idle.busy = true;
|
||||||
|
idle.lastUsed = Date.now();
|
||||||
|
return idle;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (countFor(poolKey) < MAX_WORKERS_PER_COMPANY) {
|
||||||
|
growing = growing.then(async () => {
|
||||||
|
// Re-check inside the serialized block: another caller may have grown the pool.
|
||||||
|
if (countFor(poolKey) >= MAX_WORKERS_PER_COMPANY) return;
|
||||||
|
if (pool.length >= MAX_WORKERS_TOTAL && !evictIdleOtherCompany(poolKey)) return;
|
||||||
|
try {
|
||||||
|
pool.push(await spawnWorker(poolKey, googleAuthId));
|
||||||
|
} catch (err) {
|
||||||
|
// Fail the oldest queued job for this company rather than leaving it waiting on
|
||||||
|
// a thread that will never arrive, which would deadlock the queue instead of
|
||||||
|
// surfacing the boot failure to the caller.
|
||||||
|
const index = queue.findIndex((item) => item.poolKey === poolKey);
|
||||||
|
if (index === -1) throw err;
|
||||||
|
const waiting = queue.splice(index, 1)[0];
|
||||||
|
clearTimeout(waiting.timer);
|
||||||
|
waiting.reject(err);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await growing;
|
||||||
|
} catch {
|
||||||
|
// Already reported to the queued job; continue so this caller still queues.
|
||||||
|
}
|
||||||
|
// Reserve the freshly booted thread atomically; a concurrent waiter that wakes after us
|
||||||
|
// finds none and queues instead.
|
||||||
|
const after = findIdle(poolKey);
|
||||||
|
if (after) {
|
||||||
|
after.busy = true;
|
||||||
|
after.lastUsed = Date.now();
|
||||||
|
return after;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hands one job to one thread and resolves with the worker's report.
|
||||||
|
*
|
||||||
|
* Always RESOLVES with a report-shaped object — it never rejects. The tool layer distinguishes
|
||||||
|
* outcomes by shape (`timedOut`, `error`, or a `result` key), matching what the old one-shot
|
||||||
|
* worker returned, so timeout/error mapping in the routes keeps working unchanged.
|
||||||
|
*
|
||||||
|
* @param {PoolEntry} entry
|
||||||
|
* @param {object} job Forwarded to the worker verbatim.
|
||||||
|
* @param {number} timeoutMs
|
||||||
|
* @returns {Promise<object>}
|
||||||
|
*/
|
||||||
|
function runJobOnWorker(entry, job, timeoutMs) {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const jobId = `${entry.id}:${++jobSeq}`;
|
||||||
|
let settled = false;
|
||||||
|
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
// Terminate first, then report. A snippet stuck in a native call (libvips, fetch)
|
||||||
|
// would otherwise keep burning the thread after the caller has moved on, and its
|
||||||
|
// eventual reply would arrive against an already-settled job.
|
||||||
|
try { entry.worker.terminate(); } catch { /* already closed */ }
|
||||||
|
dropWorker(entry);
|
||||||
|
finish({ result: null, stdout: '', consoleLineCount: 0, timedOut: true, timeoutMs });
|
||||||
|
}, timeoutMs);
|
||||||
|
|
||||||
|
function cleanup() {
|
||||||
|
clearTimeout(timer);
|
||||||
|
entry.worker.off('message', onMessage);
|
||||||
|
entry.worker.off('error', onError);
|
||||||
|
entry.worker.off('exit', onExit);
|
||||||
|
}
|
||||||
|
|
||||||
|
function finish(payload) {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
cleanup();
|
||||||
|
resolve(payload);
|
||||||
|
}
|
||||||
|
|
||||||
|
function onMessage(msg) {
|
||||||
|
// Ignore anything that is not THIS job's report. Under `node --watch` the dev
|
||||||
|
// server's module-loader hook posts `watch:import` notices on the same channel, so
|
||||||
|
// the first message is not necessarily our reply; job ids are matched, not order.
|
||||||
|
if (!msg || msg.__sandboxReport !== true || msg.id !== jobId) return;
|
||||||
|
finish(msg);
|
||||||
|
}
|
||||||
|
|
||||||
|
function onError(err) {
|
||||||
|
dropWorker(entry);
|
||||||
|
finish({
|
||||||
|
result: null,
|
||||||
|
stdout: '',
|
||||||
|
consoleLineCount: 0,
|
||||||
|
error: (err && err.message) ? err.message : `Sandbox worker failed: ${err?.stack || err?.name || 'unknown error'}`,
|
||||||
|
errorName: (err && err.name) || 'Error',
|
||||||
|
stack: err && typeof err.stack === 'string' ? err.stack : undefined,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function onExit(code) {
|
||||||
|
dropWorker(entry);
|
||||||
|
// Reaching here means the thread died WITHOUT reporting — a hard failure, never a
|
||||||
|
// silent empty success.
|
||||||
|
console.error(`[sandbox] worker exited without reporting (code ${code})`);
|
||||||
|
finish({
|
||||||
|
result: null,
|
||||||
|
stdout: '',
|
||||||
|
consoleLineCount: 0,
|
||||||
|
error: `Sandbox worker exited unexpectedly without reporting a result (code ${code}).`,
|
||||||
|
errorName: 'WorkerExit',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
entry.worker.on('message', onMessage);
|
||||||
|
entry.worker.on('error', onError);
|
||||||
|
entry.worker.on('exit', onExit);
|
||||||
|
|
||||||
|
try {
|
||||||
|
entry.worker.postMessage({ ...job, id: jobId });
|
||||||
|
} catch (err) {
|
||||||
|
finish({
|
||||||
|
result: null,
|
||||||
|
stdout: '',
|
||||||
|
consoleLineCount: 0,
|
||||||
|
error: (err && err.message) || 'Failed to post job to sandbox worker',
|
||||||
|
errorName: 'PostMessageError',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Starts as many queued jobs as there are idle threads for their company.
|
||||||
|
*/
|
||||||
|
function drainQueue() {
|
||||||
|
for (;;) {
|
||||||
|
const index = queue.findIndex((item) => findIdle(item.poolKey));
|
||||||
|
if (index === -1) return;
|
||||||
|
|
||||||
|
const item = queue.splice(index, 1)[0];
|
||||||
|
clearTimeout(item.timer);
|
||||||
|
const entry = findIdle(item.poolKey);
|
||||||
|
entry.busy = true;
|
||||||
|
entry.lastUsed = Date.now();
|
||||||
|
|
||||||
|
runJobOnWorker(entry, item.job, item.timeoutMs)
|
||||||
|
.then(item.resolve, item.reject)
|
||||||
|
.finally(() => {
|
||||||
|
entry.busy = false;
|
||||||
|
drainQueue();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Runs one sandbox job on a pooled thread, spawning a thread if the caps allow and otherwise
|
||||||
|
* queueing until one frees up.
|
||||||
|
*
|
||||||
|
* @param {object} options
|
||||||
|
* @param {string} options.poolKey VFS root; identifies the company (and the thread).
|
||||||
|
* @param {number} options.googleAuthId
|
||||||
|
* @param {string} options.code Source to run.
|
||||||
|
* @param {number} options.timeoutMs
|
||||||
|
* @param {string} options.dashbackToken Per-run JWT for the sandbox's `fetch()`.
|
||||||
|
* @param {object} [options.appletInput]
|
||||||
|
* @param {string} [options.appletFile]
|
||||||
|
* @returns {Promise<object>} Worker report (`result` / `timedOut` / `error` shape).
|
||||||
|
*/
|
||||||
|
export async function runSandboxJob(options) {
|
||||||
|
if (shuttingDown) {
|
||||||
|
throw new Error('Sandbox worker pool is shutting down');
|
||||||
|
}
|
||||||
|
|
||||||
|
const { poolKey, googleAuthId, code, timeoutMs, dashbackToken, appletInput, appletFile } = options;
|
||||||
|
const job = { code, timeoutMs, dashbackToken, appletInput, appletFile, poolKey, googleAuthId };
|
||||||
|
|
||||||
|
const entry = await acquireWorker(poolKey, googleAuthId);
|
||||||
|
if (entry) {
|
||||||
|
try {
|
||||||
|
return await runJobOnWorker(entry, job, timeoutMs);
|
||||||
|
} finally {
|
||||||
|
entry.busy = false;
|
||||||
|
drainQueue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
// The queue wait is bounded by the same timeout budget as execution, so a caller cannot
|
||||||
|
// wait forever behind other companies' jobs.
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
const index = queue.findIndex((item) => item.timer === timer);
|
||||||
|
if (index !== -1) queue.splice(index, 1);
|
||||||
|
resolve({
|
||||||
|
result: null,
|
||||||
|
stdout: '',
|
||||||
|
consoleLineCount: 0,
|
||||||
|
error: `Sandbox queue timed out after ${timeoutMs} ms (all workers busy).`,
|
||||||
|
errorName: 'SandboxQueueTimeout',
|
||||||
|
});
|
||||||
|
}, timeoutMs);
|
||||||
|
|
||||||
|
queue.push({ resolve, reject, timer, poolKey, job, timeoutMs });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Terminates every pooled thread and fails anything still queued.
|
||||||
|
*
|
||||||
|
* Mirrors `shutdownImageCropWorkers`: live worker threads keep the process alive after the last
|
||||||
|
* request, so `node --watch` hangs on "Waiting for graceful termination..." without this.
|
||||||
|
*
|
||||||
|
* @returns {Promise<void>}
|
||||||
|
*/
|
||||||
|
export async function shutdownJavascriptSandboxWorkers() {
|
||||||
|
shuttingDown = true;
|
||||||
|
|
||||||
|
while (queue.length > 0) {
|
||||||
|
const pending = queue.shift();
|
||||||
|
clearTimeout(pending.timer);
|
||||||
|
pending.resolve({
|
||||||
|
result: null,
|
||||||
|
stdout: '',
|
||||||
|
consoleLineCount: 0,
|
||||||
|
error: 'Sandbox worker pool is shutting down',
|
||||||
|
errorName: 'SandboxShutdown',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const entries = pool.splice(0, pool.length);
|
||||||
|
await Promise.all(
|
||||||
|
entries.map((entry) =>
|
||||||
|
Promise.resolve()
|
||||||
|
.then(() => entry.worker.terminate())
|
||||||
|
.catch(() => { /* already closed */ })
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
shuttingDown = false;
|
||||||
|
}
|
||||||
@@ -10,18 +10,15 @@ import ExcelJS from 'exceljs';
|
|||||||
import PDFDocument from 'pdfkit';
|
import PDFDocument from 'pdfkit';
|
||||||
import { createVfs } from './vfs.js';
|
import { createVfs } from './vfs.js';
|
||||||
|
|
||||||
|
// Static, per-THREAD configuration. A pooled thread serves many jobs, so per-job values
|
||||||
|
// (`code`, `appletInput`, `appletFile`, the request-scoped token and timeout) arrive as
|
||||||
|
// `runJob` messages instead — see the job loop at the bottom of this file.
|
||||||
const {
|
const {
|
||||||
code,
|
|
||||||
root,
|
root,
|
||||||
vmTimeout,
|
|
||||||
bufferLimit,
|
bufferLimit,
|
||||||
network,
|
|
||||||
dashbackUrl,
|
dashbackUrl,
|
||||||
dashbackToken,
|
|
||||||
vfsDir,
|
vfsDir,
|
||||||
googleAuthId,
|
googleAuthId,
|
||||||
appletInput,
|
|
||||||
appletFile,
|
|
||||||
} = workerData;
|
} = workerData;
|
||||||
|
|
||||||
function resolveDashbackUrl(inputUrl, baseApiUrl) {
|
function resolveDashbackUrl(inputUrl, baseApiUrl) {
|
||||||
@@ -425,7 +422,6 @@ function bridgePDFKit(vfs) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const NETWORK_GLOBALS = {
|
const NETWORK_GLOBALS = {
|
||||||
fetch: createSandboxFetch(dashbackUrl, dashbackToken),
|
|
||||||
// WebCrypto globals (bridged across the vm2 realm boundary — see bridgeCrypto above).
|
// WebCrypto globals (bridged across the vm2 realm boundary — see bridgeCrypto above).
|
||||||
// Node's `crypto` MODULE is deliberately not exposed.
|
// Node's `crypto` MODULE is deliberately not exposed.
|
||||||
...(bridgedCrypto ? { crypto: bridgedCrypto } : {}),
|
...(bridgedCrypto ? { crypto: bridgedCrypto } : {}),
|
||||||
@@ -448,46 +444,71 @@ const NETWORK_GLOBALS = {
|
|||||||
clearTimeout,
|
clearTimeout,
|
||||||
};
|
};
|
||||||
|
|
||||||
const lines = [];
|
/**
|
||||||
const push = (prefix) => (...args) => {
|
* Builds the sandbox `fetch` for ONE job.
|
||||||
let text;
|
*
|
||||||
try {
|
* `fetch` signs its `Authorization` header with the JWT minted for that job's `googleAuthId`,
|
||||||
text = args
|
* so it must not live in a module-level object shared across every company a pooled thread
|
||||||
.map((a) => {
|
* happens to serve.
|
||||||
if (typeof a === 'string') return a;
|
*
|
||||||
if (Buffer.isBuffer(a) || ArrayBuffer.isView(a) || (a && a.constructor && a.constructor.name === 'Buffer')) {
|
* @param {string} dashbackToken
|
||||||
return `<Buffer ${(a.byteLength ?? a.length ?? 0)} bytes>`;
|
*/
|
||||||
}
|
function createRunGlobals(dashbackToken) {
|
||||||
try {
|
return {
|
||||||
return JSON.stringify(a, (k, v) => {
|
fetch: createSandboxFetch(dashbackUrl, dashbackToken),
|
||||||
if (v && typeof v === 'object') {
|
...NETWORK_GLOBALS,
|
||||||
if (v.type === 'Buffer' && Array.isArray(v.data)) {
|
};
|
||||||
return `<Buffer ${v.data.length} bytes>`;
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Builds a console that appends to the given per-job line buffer.
|
||||||
|
*
|
||||||
|
* A pooled thread runs many applets, so each job needs its OWN sink — otherwise job N's stdout
|
||||||
|
* would include job N-1's output.
|
||||||
|
*
|
||||||
|
* @param {Array<string>} lines
|
||||||
|
*/
|
||||||
|
function createSandboxConsole(lines) {
|
||||||
|
const push = (prefix) => (...args) => {
|
||||||
|
let text;
|
||||||
|
try {
|
||||||
|
text = args
|
||||||
|
.map((a) => {
|
||||||
|
if (typeof a === 'string') return a;
|
||||||
|
if (Buffer.isBuffer(a) || ArrayBuffer.isView(a) || (a && a.constructor && a.constructor.name === 'Buffer')) {
|
||||||
|
return `<Buffer ${(a.byteLength ?? a.length ?? 0)} bytes>`;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return JSON.stringify(a, (k, v) => {
|
||||||
|
if (v && typeof v === 'object') {
|
||||||
|
if (v.type === 'Buffer' && Array.isArray(v.data)) {
|
||||||
|
return `<Buffer ${v.data.length} bytes>`;
|
||||||
|
}
|
||||||
|
if (Buffer.isBuffer(v) || ArrayBuffer.isView(v) || (v.constructor && v.constructor.name === 'Buffer')) {
|
||||||
|
return `<Buffer ${(v.byteLength ?? v.length ?? 0)} bytes>`;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (Buffer.isBuffer(v) || ArrayBuffer.isView(v) || (v.constructor && v.constructor.name === 'Buffer')) {
|
return v;
|
||||||
return `<Buffer ${(v.byteLength ?? v.length ?? 0)} bytes>`;
|
});
|
||||||
}
|
} catch {
|
||||||
}
|
return String(a);
|
||||||
return v;
|
}
|
||||||
});
|
})
|
||||||
} catch {
|
.join(' ');
|
||||||
return String(a);
|
} catch {
|
||||||
}
|
text = String(args);
|
||||||
})
|
}
|
||||||
.join(' ');
|
lines.push(prefix + text);
|
||||||
} catch {
|
};
|
||||||
text = String(args);
|
return {
|
||||||
}
|
log: push(''),
|
||||||
lines.push(prefix + text);
|
info: push(''),
|
||||||
};
|
debug: push(''),
|
||||||
const sandboxConsole = {
|
warn: push('warn: '),
|
||||||
log: push(''),
|
error: push('error: '),
|
||||||
info: push(''),
|
trace: push('trace: '),
|
||||||
debug: push(''),
|
};
|
||||||
warn: push('warn: '),
|
}
|
||||||
error: push('error: '),
|
|
||||||
trace: push('trace: '),
|
|
||||||
};
|
|
||||||
|
|
||||||
const MAX_ITEMS = 200;
|
const MAX_ITEMS = 200;
|
||||||
|
|
||||||
@@ -527,37 +548,53 @@ function sanitize(value, depth = 0) {
|
|||||||
return { type: t };
|
return { type: t };
|
||||||
}
|
}
|
||||||
|
|
||||||
// Every genuine sandbox report is tagged with `__sandboxReport: true`.
|
/**
|
||||||
//
|
* Posts a job report back to the pool.
|
||||||
// This is REQUIRED, not cosmetic: when the server runs under `node --watch` (see the `dev`
|
*
|
||||||
// script), Node installs a module-loader hook that posts `watch:import` notices over the SAME
|
* Every report is tagged with the job id it answers AND `__sandboxReport: true`.
|
||||||
// parentPort channel the worker reports on. Without this tag the parent cannot tell a real
|
*
|
||||||
// report apart from loader chatter and will resolve on the wrong message.
|
* The tag is REQUIRED, not cosmetic: when the server runs under `node --watch` (see the `dev`
|
||||||
function report(payload) {
|
* script), Node installs a module-loader hook that posts `watch:import` notices over the SAME
|
||||||
const message = { __sandboxReport: true, ...payload };
|
* parentPort channel the worker reports on. The pool matches the `id` to route a reply to the
|
||||||
|
* right job; without the tag it cannot tell a real report apart from loader chatter.
|
||||||
|
*
|
||||||
|
* @param {string} id
|
||||||
|
* @param {object} payload
|
||||||
|
*/
|
||||||
|
function report(id, payload) {
|
||||||
try {
|
try {
|
||||||
parentPort.postMessage(message);
|
parentPort.postMessage({ __sandboxReport: true, id, ...payload });
|
||||||
} catch {
|
} catch {
|
||||||
try {
|
try {
|
||||||
parentPort.postMessage({
|
parentPort.postMessage({ __sandboxReport: true, id, result: null, error: 'Sandbox produced a result that could not be serialized.', errorName: 'SerializationError' });
|
||||||
__sandboxReport: true,
|
|
||||||
result: null,
|
|
||||||
error: 'Sandbox produced a result that could not be serialized.',
|
|
||||||
errorName: 'SerializationError',
|
|
||||||
stdout: lines.join('\n'),
|
|
||||||
consoleLineCount: lines.length,
|
|
||||||
});
|
|
||||||
} catch { /* thread exiting */ }
|
} catch { /* thread exiting */ }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void (async () => {
|
/**
|
||||||
try {
|
* Runs ONE job posted by the pool. Never throws: every outcome is reported via {@link report}.
|
||||||
|
*
|
||||||
|
* @param {object} job
|
||||||
|
* @param {string} job.id Opaque job id; echoed back on the report.
|
||||||
|
* @param {string} job.code Source to run.
|
||||||
|
* @param {string} job.dashbackToken Per-job JWT used to sign sandbox `fetch()` calls.
|
||||||
|
* @param {object} [job.appletInput] Value passed as `applet_input` when `appletFile` is set.
|
||||||
|
* @param {string} [job.appletFile] VFS path of the applet, which switches on the applet protocol.
|
||||||
|
*/
|
||||||
|
async function runJob(job) {
|
||||||
|
const code = job.code;
|
||||||
|
const appletInput = job.appletInput;
|
||||||
|
const appletFile = job.appletFile;
|
||||||
|
|
||||||
const vfs = createVfs(vfsDir, { googleAuthId });
|
const vfs = createVfs(vfsDir, { googleAuthId });
|
||||||
|
|
||||||
const bridgedExcel = bridgeExcelJS(vfs);
|
const bridgedExcel = bridgeExcelJS(vfs);
|
||||||
const bridgedPDF = bridgePDFKit(vfs);
|
const bridgedPDF = bridgePDFKit(vfs);
|
||||||
|
|
||||||
|
// One console sink per job so pooled runs cannot see each other's stdout.
|
||||||
|
const lines = [];
|
||||||
|
const sandboxConsole = createSandboxConsole(lines);
|
||||||
|
|
||||||
const vm = new NodeVM({
|
const vm = new NodeVM({
|
||||||
console: 'off',
|
console: 'off',
|
||||||
sandbox: {
|
sandbox: {
|
||||||
@@ -572,7 +609,7 @@ try {
|
|||||||
pdfkit: bridgedPDF,
|
pdfkit: bridgedPDF,
|
||||||
validateLabelObjects: (objects) => validateObjects(JSON.parse(JSON.stringify(objects))),
|
validateLabelObjects: (objects) => validateObjects(JSON.parse(JSON.stringify(objects))),
|
||||||
validateLabelObject: (object) => validateObject(JSON.parse(JSON.stringify(object))),
|
validateLabelObject: (object) => validateObject(JSON.parse(JSON.stringify(object))),
|
||||||
...NETWORK_GLOBALS,
|
...createRunGlobals(job.dashbackToken),
|
||||||
},
|
},
|
||||||
require: {
|
require: {
|
||||||
root,
|
root,
|
||||||
@@ -581,7 +618,7 @@ try {
|
|||||||
},
|
},
|
||||||
nesting: false,
|
nesting: false,
|
||||||
bufferAllocLimit: bufferLimit,
|
bufferAllocLimit: bufferLimit,
|
||||||
timeout: vmTimeout,
|
timeout: job.timeoutMs,
|
||||||
});
|
});
|
||||||
|
|
||||||
const started = Date.now();
|
const started = Date.now();
|
||||||
@@ -602,52 +639,67 @@ try {
|
|||||||
value = value(appletInput);
|
value = value(appletInput);
|
||||||
}
|
}
|
||||||
if (value && typeof value.then === 'function') {
|
if (value && typeof value.then === 'function') {
|
||||||
const settleTimeout = Math.max(1000, vmTimeout - 500);
|
const settleTimeout = Math.max(1000, job.timeoutMs - 500);
|
||||||
value = await Promise.race([
|
value = await Promise.race([
|
||||||
value,
|
value,
|
||||||
new Promise((_, reject) =>
|
new Promise((_, reject) =>
|
||||||
setTimeout(() => reject(new Error('Async snippet did not settle before the timeout.')), settleTimeout)
|
setTimeout(() => reject(new Error('Async snippet did not settle before the timeout.')), settleTimeout)
|
||||||
),
|
),
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const rawMsg = (err && err.message) || String(err);
|
const rawMsg = (err && err.message) || String(err);
|
||||||
failed = {
|
failed = {
|
||||||
error: rawMsg,
|
error: rawMsg,
|
||||||
errorName: (err && err.name) || 'Error',
|
errorName: (err && err.name) || 'Error',
|
||||||
cause: err && err.cause ? ((err.cause && err.cause.message) || String(err.cause)) : undefined,
|
cause: err && err.cause ? ((err.cause && err.cause.message) || String(err.cause)) : undefined,
|
||||||
stack: err && typeof err.stack === 'string' ? err.stack : undefined,
|
stack: err && typeof err.stack === 'string' ? err.stack : undefined,
|
||||||
};
|
};
|
||||||
// Snippets run in a CommonJS wrapper, so bare top-level `await` is a syntax error.
|
// Snippets run in a CommonJS wrapper, so bare top-level `await` is a syntax error.
|
||||||
// That is the single most common mistake, so say exactly how to fix it.
|
// That is the single most common mistake, so say exactly how to fix it.
|
||||||
if (err instanceof SyntaxError && /await is only valid|Unexpected reserved word/i.test(rawMsg)) {
|
if (err instanceof SyntaxError && /await is only valid|Unexpected reserved word/i.test(rawMsg)) {
|
||||||
failed.error =
|
failed.error =
|
||||||
'Top-level `await` is not allowed: snippets run in a CommonJS (non-ESM) wrapper. ' +
|
'Top-level `await` is not allowed: snippets run in a CommonJS (non-ESM) wrapper. ' +
|
||||||
'Wrap async code in an async IIFE and assign the promise to module.exports:\n' +
|
'Wrap async code in an async IIFE and assign the promise to module.exports:\n' +
|
||||||
' module.exports = (async () => { const res = await fetch("/company"); return res.json(); })();';
|
' module.exports = (async () => { const res = await fetch("/company"); return res.json(); })();';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (failed) {
|
if (failed) {
|
||||||
console.error('[javascript-worker] snippet threw:', failed.error, '\n', failed.stack || '(no stack)');
|
console.error('[javascript-worker] snippet threw:', failed.error, '\n', failed.stack || '(no stack)');
|
||||||
report({ result: null, ...done(), ...failed });
|
report(job.id, { result: null, ...done(), ...failed });
|
||||||
} else {
|
} else {
|
||||||
// `result` MUST always be present: if the key were absent, the parent's
|
// `result` MUST always be present: if the key were absent, the parent's
|
||||||
// JSON.stringify() drops it and the tool response degenerates to a bare
|
// JSON.stringify() drops it and the tool response degenerates to a bare
|
||||||
// echo of the submitted source with no sign of execution.
|
// echo of the submitted source with no sign of execution.
|
||||||
report({ result: sanitize(value), ...done() });
|
report(job.id, { result: sanitize(value), ...done() });
|
||||||
}
|
}
|
||||||
} catch (err) {
|
|
||||||
// Setup-level failure (bad VFS dir, thrown import, etc.) — log it here too, because the
|
|
||||||
// parent only sees a message and the worker's stderr is otherwise swallowed.
|
|
||||||
console.error('[javascript-worker] fatal setup error:', (err && err.message) || String(err), '\n', (err && err.stack) || '(no stack)');
|
|
||||||
report({
|
|
||||||
result: null,
|
|
||||||
stdout: lines.join('\n'),
|
|
||||||
consoleLineCount: lines.length,
|
|
||||||
error: (err && err.message) || String(err),
|
|
||||||
errorName: (err && err.name) || 'Error',
|
|
||||||
stack: err && typeof err.stack === 'string' ? err.stack : undefined,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
})();
|
|
||||||
|
// Pooled job loop. Imports (sharp, exceljs, pdfkit, libdashstore, …) are paid ONCE for the
|
||||||
|
// lifetime of the thread; each subsequent job only builds a fresh NodeVM and runs the snippet,
|
||||||
|
// which is the ~2 ms part. Jobs are serialized (one `await` per message) so a pooled thread
|
||||||
|
// never interleaves two applets against the same VFS.
|
||||||
|
parentPort.on('message', (job) => {
|
||||||
|
void (async () => {
|
||||||
|
try {
|
||||||
|
await runJob(job);
|
||||||
|
} catch (err) {
|
||||||
|
// Setup-level failure (bad VFS dir, thrown bridge, etc.) — log it here too, because
|
||||||
|
// the pool only sees a message and this thread's stderr is otherwise swallowed.
|
||||||
|
console.error('[javascript-worker] fatal setup error:', (err && err.message) || String(err), '\n', (err && err.stack) || '(no stack)');
|
||||||
|
report(job?.id, {
|
||||||
|
result: null,
|
||||||
|
stdout: '',
|
||||||
|
consoleLineCount: 0,
|
||||||
|
error: (err && err.message) || String(err),
|
||||||
|
errorName: (err && err.name) || 'Error',
|
||||||
|
stack: err && typeof err.stack === 'string' ? err.stack : undefined,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Ready handshake. All imports above have resolved by the time this module body executes, so
|
||||||
|
// signalling now tells the pool it can hand this thread a job without racing the import phase.
|
||||||
|
parentPort.postMessage({ ready: true });
|
||||||
|
|||||||
@@ -1,109 +1,198 @@
|
|||||||
import { Worker } from 'worker_threads';
|
|
||||||
import { fileURLToPath } from 'url';
|
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
import jwt from 'jsonwebtoken';
|
import jwt from 'jsonwebtoken';
|
||||||
import { defineTool, ToolError } from 'libllmorbiter';
|
import { defineTool, ToolError } from 'libllmorbiter';
|
||||||
import env from '../config/env.js';
|
import env from '../config/env.js';
|
||||||
import { createVfs, prepareVfs, syncVfsToDb } from './vfs.js';
|
import { createVfs, prepareVfs, syncVfsToDb } from './vfs.js';
|
||||||
|
import { runSandboxJob } from '../services/javascriptSandbox.js';
|
||||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
||||||
const ROOT_DIR = path.resolve(__dirname, '../..');
|
|
||||||
const WORKER_PATH = fileURLToPath(new URL('./javascript-worker.js', import.meta.url));
|
|
||||||
|
|
||||||
const DEFAULT_TIMEOUT_MS = 20_000;
|
const DEFAULT_TIMEOUT_MS = 20_000;
|
||||||
const MAX_TIMEOUT_MS = 60_000;
|
const MAX_TIMEOUT_MS = 60_000;
|
||||||
const DEFAULT_MAX_OUTPUT = 20_000;
|
const DEFAULT_MAX_OUTPUT = 20_000;
|
||||||
const DEFAULT_BUFFER_LIMIT = 32 * 1024 * 1024;
|
|
||||||
|
|
||||||
// Keys already logged as benign non-sandbox worker chatter. Under `node --watch` the loader
|
// Column types an applet may declare in its `@applet-output-schema` header. Kept deliberately
|
||||||
// emits `watch:import` / `watch:require` notices on the worker's parentPort for EVERY import,
|
// small: each one maps to a JS-coercion rule the applet author can reason about, and the
|
||||||
// so warning per message floods the log. We warn once per distinct key instead.
|
// frontend can render without guessing.
|
||||||
const WATCH_NOISE_WARNED = new Set();
|
const APPLET_COLUMN_TYPES = new Set(['string', 'integer', 'number', 'boolean', 'date']);
|
||||||
|
|
||||||
function runInWorker({ code, timeoutMs, vfsRoot, googleAuthId, appletInput, appletFile }) {
|
// Semicolon-separated columns, matching the spreadsheet exports in `src/util/*Csv.js`.
|
||||||
return new Promise((resolve) => {
|
const APPLET_CSV_SEPARATOR = ';';
|
||||||
let settled = false;
|
|
||||||
const token = jwt.sign({ id: googleAuthId }, env.jwtSecret, { expiresIn: '5m' });
|
|
||||||
|
|
||||||
const worker = new Worker(WORKER_PATH, {
|
/**
|
||||||
workerData: {
|
* Parse the applet output header into typed CSV columns.
|
||||||
code,
|
*
|
||||||
root: ROOT_DIR,
|
* The header carries the COLUMN NAMES AND TYPES, e.g. `id:integer;name:string;active:boolean`.
|
||||||
vmTimeout: timeoutMs,
|
* A bare `name` (no `:type`) is accepted and defaults to `string`. Types are limited to
|
||||||
bufferLimit: DEFAULT_BUFFER_LIMIT,
|
* {@link APPLET_COLUMN_TYPES} so a typo surfaces here instead of at render time.
|
||||||
dashbackUrl: `http://127.0.0.1:${env.port}/api`,
|
*
|
||||||
dashbackToken: token,
|
* @param {string} header raw text after the `// @applet-output-schema ` marker
|
||||||
vfsDir: vfsRoot,
|
* @returns {Array<{ name: string, type: string }>}
|
||||||
googleAuthId,
|
*/
|
||||||
appletInput,
|
function parseTypedCsvColumns(header) {
|
||||||
appletFile,
|
const text = String(header || '').trim();
|
||||||
},
|
if (!text) {
|
||||||
});
|
throw new ToolError(
|
||||||
|
'Applet output schema must declare typed CSV columns, e.g. ' +
|
||||||
|
'`@applet-output-schema id:integer;name:string`.'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const finish = (payload) => {
|
const columns = [];
|
||||||
if (settled) return;
|
const seen = new Set();
|
||||||
settled = true;
|
for (const raw of text.split(APPLET_CSV_SEPARATOR)) {
|
||||||
clearTimeout(timer);
|
const field = raw.trim();
|
||||||
try { worker.terminate(); } catch { /* already closed */ }
|
if (!field) continue;
|
||||||
resolve(payload);
|
|
||||||
};
|
|
||||||
|
|
||||||
const timer = setTimeout(() => {
|
const sep = field.lastIndexOf(':');
|
||||||
console.error(`[javascript_execute] sandbox worker timed out after ${timeoutMs} ms`);
|
const name = (sep === -1 ? field : field.slice(0, sep)).trim();
|
||||||
finish({ result: null, stdout: '', consoleLineCount: 0, timedOut: true, timeoutMs });
|
const type = (sep === -1 ? 'string' : field.slice(sep + 1).trim().toLowerCase());
|
||||||
}, timeoutMs);
|
|
||||||
|
|
||||||
worker.on('message', (msg) => {
|
if (!name) {
|
||||||
// Ignore anything that is not a genuine sandbox report. Running the server under
|
throw new ToolError(`Applet output column name is empty in "${field}".`);
|
||||||
// `node --watch` (the `dev` script) installs a module-loader hook that posts
|
}
|
||||||
// `watch:import` notices on this same channel; treating the first message as the
|
if (!APPLET_COLUMN_TYPES.has(type)) {
|
||||||
// result made the sandbox appear to run for 0ms and return nothing.
|
throw new ToolError(
|
||||||
if (!msg || msg.__sandboxReport !== true) {
|
`Applet output column "${name}" has unknown type "${type}". ` +
|
||||||
// Those loader notices fire many times per run, so warn ONCE per key rather
|
`Use one of: ${[...APPLET_COLUMN_TYPES].join(', ')}.`
|
||||||
// than on every message — expected, benign noise while running under --watch.
|
);
|
||||||
for (const key of Object.keys(msg || {})) {
|
}
|
||||||
if (!WATCH_NOISE_WARNED.has(key)) {
|
if (seen.has(name)) {
|
||||||
WATCH_NOISE_WARNED.add(key);
|
throw new ToolError(`Applet output column "${name}" is declared twice.`);
|
||||||
console.warn(
|
}
|
||||||
`[javascript_execute] ignoring non-sandbox worker message "${key}" ` +
|
seen.add(name);
|
||||||
'(expected under `node --watch`; logged once per key).'
|
columns.push({ name, type });
|
||||||
);
|
}
|
||||||
}
|
|
||||||
|
if (columns.length === 0) {
|
||||||
|
throw new ToolError('Applet output schema must declare at least one column.');
|
||||||
|
}
|
||||||
|
return columns;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Quote one CSV field (RFC 4180): wrap when it contains the separator, a quote, or a newline.
|
||||||
|
* @param {unknown} value
|
||||||
|
* @returns {string}
|
||||||
|
*/
|
||||||
|
function encodeCsvField(value) {
|
||||||
|
const text = value == null ? '' : String(value);
|
||||||
|
if (text.includes(APPLET_CSV_SEPARATOR) || text.includes('"') || /[\r\n]/.test(text)) {
|
||||||
|
return `"${text.replace(/"/g, '""')}"`;
|
||||||
|
}
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Coerce a returned cell to its declared type and report what happened.
|
||||||
|
*
|
||||||
|
* Returns `null` when the value cannot be coerced (`undefined`, `null`, `NaN`, unparseable
|
||||||
|
* date). The caller turns that into an empty CSV cell, so a missing value never aborts the run.
|
||||||
|
*
|
||||||
|
* @param {unknown} value
|
||||||
|
* @param {string} type
|
||||||
|
* @returns {{ ok: true, value: string | number | boolean } | { ok: false }}
|
||||||
|
*/
|
||||||
|
function coerceAppletCell(value, type) {
|
||||||
|
if (value === undefined || value === null) return { ok: false };
|
||||||
|
|
||||||
|
switch (type) {
|
||||||
|
case 'string':
|
||||||
|
return { ok: true, value: typeof value === 'object' ? JSON.stringify(value) : String(value) };
|
||||||
|
case 'boolean':
|
||||||
|
if (typeof value === 'boolean') return { ok: true, value };
|
||||||
|
if (value === 1 || value === 0) return { ok: true, value: value === 1 };
|
||||||
|
if (typeof value === 'string' && /^(true|false)$/i.test(value.trim())) {
|
||||||
|
return { ok: true, value: value.trim().toLowerCase() === 'true' };
|
||||||
|
}
|
||||||
|
return { ok: false };
|
||||||
|
case 'integer':
|
||||||
|
case 'number': {
|
||||||
|
const num = typeof value === 'number' ? value : Number(String(value).trim());
|
||||||
|
if (!Number.isFinite(num)) return { ok: false };
|
||||||
|
return { ok: true, value: type === 'integer' ? Math.trunc(num) : num };
|
||||||
|
}
|
||||||
|
case 'date': {
|
||||||
|
const date = value instanceof Date ? value : new Date(value);
|
||||||
|
if (Number.isNaN(date.getTime())) return { ok: false };
|
||||||
|
return { ok: true, value: date.toISOString() };
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return { ok: false };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Serialize an applet's return value into typed CSV keyed by its declared columns.
|
||||||
|
*
|
||||||
|
* The applet returns rows as objects (or arrays). Only declared columns are emitted, in header
|
||||||
|
* order; unknown keys are dropped and missing ones become empty cells. Scalars and key
|
||||||
|
* information that belong to the row itself are surfaced as `warnings` rather than failing.
|
||||||
|
*
|
||||||
|
* @param {unknown} result value returned by the applet's exported function
|
||||||
|
* @param {Array<{ name: string, type: string }>} columns
|
||||||
|
* @returns {{ csv: string, rows: number, columns: Array<{ name: string, type: string }>, warnings: string[] }}
|
||||||
|
*/
|
||||||
|
function toTypedCsv(result, columns) {
|
||||||
|
const warnings = [];
|
||||||
|
|
||||||
|
let rows;
|
||||||
|
if (result === undefined || result === null) {
|
||||||
|
rows = [];
|
||||||
|
} else if (Array.isArray(result)) {
|
||||||
|
rows = result;
|
||||||
|
} else if (typeof result === 'object') {
|
||||||
|
// A single object is one row — the common "return the totals" case.
|
||||||
|
rows = [result];
|
||||||
|
} else {
|
||||||
|
rows = [result];
|
||||||
|
warnings.push(
|
||||||
|
`Applet returned a ${typeof result}, which has no columns to map onto; ` +
|
||||||
|
'the value was placed in the first column.'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let coercions = 0;
|
||||||
|
const body = rows.map((row) => {
|
||||||
|
const isContainer = row && typeof row === 'object' && !Array.isArray(row);
|
||||||
|
return columns
|
||||||
|
.map((column, index) => {
|
||||||
|
const value = isContainer ? row[column.name] : index === 0 ? row : undefined;
|
||||||
|
const coerced = coerceAppletCell(value, column.type);
|
||||||
|
if (!coerced.ok) {
|
||||||
|
if (value !== undefined && value !== null) coercions += 1;
|
||||||
|
return '';
|
||||||
}
|
}
|
||||||
return;
|
return encodeCsvField(coerced.value);
|
||||||
}
|
})
|
||||||
finish(msg);
|
.join(APPLET_CSV_SEPARATOR);
|
||||||
});
|
});
|
||||||
// A worker 'error' fires BEFORE any message when the thread fails to boot (e.g. an
|
|
||||||
// import throws at startup). ALWAYS send a truthy `error` plus the full stack so the
|
if (coercions > 0) {
|
||||||
// parent can throw with something actionable; an empty `err.message` must never
|
warnings.push(`${coercions} cell(s) did not match their declared column type and were left empty.`);
|
||||||
// degrade into a silent, result-less success payload.
|
}
|
||||||
worker.on('error', (err) => finish({
|
|
||||||
result: null,
|
const csv = [columns.map((c) => encodeCsvField(c.name)).join(APPLET_CSV_SEPARATOR), ...body].join('\n');
|
||||||
stdout: '',
|
return { csv, rows: body.length, columns, warnings };
|
||||||
consoleLineCount: 0,
|
}
|
||||||
error: (err && err.message)
|
|
||||||
? err.message
|
/**
|
||||||
: `Sandbox worker failed to start: ${err?.stack || err?.name || 'unknown error'}`,
|
* Runs one snippet/applet on the shared sandbox pool.
|
||||||
errorName: (err && err.name) || 'Error',
|
*
|
||||||
stack: err && typeof err.stack === 'string' ? err.stack : undefined,
|
* Threads are keyed by VFS root so a reused thread keeps the right company's `vfsDir`, and the
|
||||||
}));
|
* per-run JWT is minted here because it is scoped to THIS `googleAuthId`.
|
||||||
worker.on('exit', (code) => {
|
*
|
||||||
// `finish` is idempotent via `settled`, so a message that already resolved the
|
* @param {{ code: string, timeoutMs: number, vfsRoot: string, googleAuthId: number, appletInput?: object, appletFile?: string }} ctx
|
||||||
// promise is never overwritten. Reaching here means the worker died WITHOUT
|
*/
|
||||||
// reporting — that is a hard failure, so log it and always carry a truthy error.
|
function runInWorker({ code, timeoutMs, vfsRoot, googleAuthId, appletInput, appletFile }) {
|
||||||
if (!settled) {
|
const dashbackToken = jwt.sign({ id: googleAuthId }, env.jwtSecret, { expiresIn: '5m' });
|
||||||
console.error(`[javascript_execute] sandbox worker exited without reporting (code ${code})`);
|
return runSandboxJob({
|
||||||
}
|
poolKey: vfsRoot,
|
||||||
finish({
|
googleAuthId,
|
||||||
result: null,
|
code,
|
||||||
stdout: '',
|
timeoutMs,
|
||||||
consoleLineCount: 0,
|
dashbackToken,
|
||||||
error: `Sandbox worker exited unexpectedly without reporting a result (code ${code}).`,
|
appletInput,
|
||||||
errorName: 'WorkerExit',
|
appletFile,
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -145,27 +234,24 @@ function readApplet(vfsRoot, appletvfsfile) {
|
|||||||
throw new ToolError(
|
throw new ToolError(
|
||||||
'Applet metadata must be on the first two lines: ' +
|
'Applet metadata must be on the first two lines: ' +
|
||||||
'`// @applet-input-schema {JSON Schema}` followed by ' +
|
'`// @applet-input-schema {JSON Schema}` followed by ' +
|
||||||
'`// @applet-output-schema {JSON Schema}`.'
|
'`// @applet-output-schema name:type;name:type` (typed CSV columns).'
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let inputSchema;
|
let inputSchema;
|
||||||
let outputSchema;
|
|
||||||
try {
|
try {
|
||||||
inputSchema = JSON.parse(inputHeader[1]);
|
inputSchema = JSON.parse(inputHeader[1]);
|
||||||
outputSchema = JSON.parse(outputHeader[1]);
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
throw new ToolError(`Applet metadata must contain valid JSON schemas: ${err.message}`);
|
throw new ToolError(`Applet input schema must be valid JSON: ${err.message}`);
|
||||||
}
|
}
|
||||||
if (!inputSchema || typeof inputSchema !== 'object' || Array.isArray(inputSchema) ||
|
if (!inputSchema || typeof inputSchema !== 'object' || Array.isArray(inputSchema) ||
|
||||||
inputSchema.type !== 'object') {
|
inputSchema.type !== 'object') {
|
||||||
throw new ToolError('Applet input schema must be a JSON Schema with type "object".');
|
throw new ToolError('Applet input schema must be a JSON Schema with type "object".');
|
||||||
}
|
}
|
||||||
if (!outputSchema || typeof outputSchema !== 'object' || Array.isArray(outputSchema)) {
|
|
||||||
throw new ToolError('Applet output schema must be a JSON Schema object.');
|
|
||||||
}
|
|
||||||
|
|
||||||
return source;
|
const outputColumns = parseTypedCsvColumns(outputHeader[1]);
|
||||||
|
|
||||||
|
return { source, outputColumns };
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatSandboxError(code, report) {
|
function formatSandboxError(code, report) {
|
||||||
@@ -188,7 +274,9 @@ export function createJavascriptExecuteTool({ googleAuthId }) {
|
|||||||
description:
|
description:
|
||||||
'Execute JS in an isolated worker sandbox using either inline `code` or a VFS applet. Provide exactly one of `code` and `appletvfsfile`. ' +
|
'Execute JS in an isolated worker sandbox using either inline `code` or a VFS applet. Provide exactly one of `code` and `appletvfsfile`. ' +
|
||||||
'Applets are the preferred method when the first execution may throw: save a `.applet.js` file in the VFS so it can be inspected and further edited with `patch`, then call it by VFS path. ' +
|
'Applets are the preferred method when the first execution may throw: save a `.applet.js` file in the VFS so it can be inspected and further edited with `patch`, then call it by VFS path. ' +
|
||||||
'Applet files must start with `// @applet-input-schema {JSON Schema}` and `// @applet-output-schema {JSON Schema}` on their first two lines. The input schema describes the GUI form passed as `applet_input`; the output schema describes the exported function result. Export a function that receives `applet_input`. ' +
|
'Applet files must start with `// @applet-input-schema {JSON Schema}` on line 1 and `// @applet-output-schema <columns>` on line 2. ' +
|
||||||
|
'The input schema describes the GUI form passed as `applet_input`. The output line declares the RESULT COLUMNS as typed CSV: `name:type` pairs separated by `;` (e.g. `id:integer;name:string;active:boolean`); a bare `name` defaults to string, and the allowed types are string, integer, number, boolean and date. ' +
|
||||||
|
'Applets return typed CSV, NOT JSON: export a function that receives `applet_input` and returns a row object, or an array of row objects, whose keys are the declared columns (e.g. `return [{ id: 1, name: "Kaffee" }]`). Only declared columns are emitted, in header order; missing keys become empty cells. The tool result then carries `result` as a `;`-separated CSV string with a header row, plus `columns` (name/type) and `rows` (data row count). Returning a single object yields one data row. ' +
|
||||||
'Inline code: assign `module.exports` to return value (sync value or Promise; wrap async in IIFE: `module.exports = (async () => { ... })()`). ' +
|
'Inline code: assign `module.exports` to return value (sync value or Promise; wrap async in IIFE: `module.exports = (async () => { ... })()`). ' +
|
||||||
'REST API: `await fetch("/path")` (authenticated). ' +
|
'REST API: `await fetch("/path")` (authenticated). ' +
|
||||||
'VFS: directly use global `fs.readFileSync`/`writeFileSync` for binary artifacts/buffers/images (`image/` is read-only; paths must be relative paths, root is not allowed). VFS can host a static website (HTML, CSS, JS, images, etc.). ' +
|
'VFS: directly use global `fs.readFileSync`/`writeFileSync` for binary artifacts/buffers/images (`image/` is read-only; paths must be relative paths, root is not allowed). VFS can host a static website (HTML, CSS, JS, images, etc.). ' +
|
||||||
@@ -209,11 +297,11 @@ export function createJavascriptExecuteTool({ googleAuthId }) {
|
|||||||
},
|
},
|
||||||
appletvfsfile: {
|
appletvfsfile: {
|
||||||
type: 'string',
|
type: 'string',
|
||||||
description: 'Optional VFS-relative path to a .applet.js file. Provide exactly one of this and code. The file must begin with the two applet JSON Schema metadata lines described in the tool description.',
|
description: 'Optional VFS-relative path to a .applet.js file. Provide exactly one of this and code. The file must begin with the two applet metadata lines (`@applet-input-schema` JSON Schema, then `@applet-output-schema` typed CSV columns like `id:integer;name:string`).',
|
||||||
},
|
},
|
||||||
applet_input: {
|
applet_input: {
|
||||||
type: 'object',
|
type: 'object',
|
||||||
description: 'Input values passed to the exported function when running appletvfsfile. Omit to pass an empty object.',
|
description: 'Input values passed to the exported function when running appletvfsfile. Omit to pass an empty object. The applet returns typed CSV rows mapped onto its `@applet-output-schema` columns.',
|
||||||
additionalProperties: true,
|
additionalProperties: true,
|
||||||
},
|
},
|
||||||
timeout: {
|
timeout: {
|
||||||
@@ -257,7 +345,8 @@ export function createJavascriptExecuteTool({ googleAuthId }) {
|
|||||||
|
|
||||||
const vfsRoot = await prepareVfs({ googleAuthId });
|
const vfsRoot = await prepareVfs({ googleAuthId });
|
||||||
const isApplet = hasApplet;
|
const isApplet = hasApplet;
|
||||||
const source = isApplet ? readApplet(vfsRoot, appletvfsfile) : code;
|
const applet = isApplet ? readApplet(vfsRoot, appletvfsfile) : null;
|
||||||
|
const source = isApplet ? applet.source : code;
|
||||||
let report;
|
let report;
|
||||||
try {
|
try {
|
||||||
report = await runInWorker({
|
report = await runInWorker({
|
||||||
@@ -314,6 +403,15 @@ export function createJavascriptExecuteTool({ googleAuthId }) {
|
|||||||
// Always present, even when empty, so callers can rely on the shape.
|
// Always present, even when empty, so callers can rely on the shape.
|
||||||
stdout,
|
stdout,
|
||||||
};
|
};
|
||||||
|
// Applets return typed CSV, not JSON: the declared columns turn the returned rows
|
||||||
|
// into a `;`-separated table. Inline code keeps returning its raw value.
|
||||||
|
if (isApplet) {
|
||||||
|
const table = toTypedCsv(report.result, applet.outputColumns);
|
||||||
|
result.result = table.csv;
|
||||||
|
result.rows = table.rows;
|
||||||
|
result.columns = table.columns;
|
||||||
|
if (table.warnings.length) result.warnings = table.warnings;
|
||||||
|
}
|
||||||
if (truncated) {
|
if (truncated) {
|
||||||
result.truncated = true;
|
result.truncated = true;
|
||||||
result.truncationNote = `Console output longer than ${maxOutput} chars was capped.`;
|
result.truncationNote = `Console output longer than ${maxOutput} chars was capped.`;
|
||||||
|
|||||||
Reference in New Issue
Block a user