From e7884c03ef1592f80dfcb19ef288dfedadd4c287 Mon Sep 17 00:00:00 2001 From: seb Date: Wed, 28 Oct 2026 01:46:51 +0100 Subject: [PATCH] sharp by wasm --- docs/api/routes/vfs/exec.md | 2 +- package-lock.json | 35 +++- package.json | 2 +- scripts/testImageCrop.js | 69 +++----- src/api/routes/serper.js | 33 ++-- src/api/routes/vfs.js | 101 ++++++------ src/api/routes/vfsExec.js | 2 +- src/defaultSkills/applet.SKILL | 2 +- src/services/companyImport/imageHasher.js | 21 +-- src/services/imageCrop.js | 26 ++- src/services/imageCropWorker.js | 186 ++++++++++++++-------- src/services/javascriptSandbox.js | 6 +- src/services/labelThumb.js | 102 ++++++++++-- src/services/manifestPdf.js | 8 +- src/tools/analyze-image.js | 83 ++++++---- src/tools/edit-image.js | 56 +++++-- src/tools/javascript-worker.js | 76 ++------- src/tools/prompts.js | 2 +- src/tools/search-images.js | 21 ++- src/v1/routes/pimage.js | 61 ++++--- 20 files changed, 530 insertions(+), 364 deletions(-) diff --git a/docs/api/routes/vfs/exec.md b/docs/api/routes/vfs/exec.md index 63a79fb..c6bd27f 100644 --- a/docs/api/routes/vfs/exec.md +++ b/docs/api/routes/vfs/exec.md @@ -20,7 +20,7 @@ Applets return typed CSV, not JSON: `result` is a `;`-separated CSV string with response also carries `columns` (name/type) and `rows` (data row count). Only declared columns are emitted, in header order; missing values become empty cells. Execution runs in the same isolated worker sandbox as the `javascript_execute` tool — including authenticated `fetch()` against this API, -`fs`/`vfs`, `sharp`, ExcelJS, PDFKit and WebCrypto globals. +`fs`/`vfs`, `optimizeImage`, ExcelJS, PDFKit and WebCrypto globals. The path is taken from the URL (like `DELETE /vfs/{*path}`); the run arguments are the JSON body. Path traversal (`..`), null bytes, and backslashes are rejected with HTTP 400, as is a path that does diff --git a/package-lock.json b/package-lock.json index a6a33c6..59eb48c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,10 +31,10 @@ "react": "^19.3.1", "react-canvas-label-editor": "github:sebseb7/react-canvas-label-editor", "selfsigned": "^5.5.0", - "sharp": "^0.35.5", "skelehash-wasm": "^0.1.1", "swagger-ui-dist": "^5.33.3", "vm2": "^3.12.2", + "wasm-image-optimization": "^2.0.13", "zod": "^4.6.7" }, "devDependencies": { @@ -53,9 +53,9 @@ "chalk": "^6.0.1", "dotenv": "^18.0.8", "pdfconvert-wasm": "^0.3.0", - "sharp": "^0.35.5", "skelehash-wasm": "^0.1.1", - "thumbhash": "^0.1.1" + "thumbhash": "^0.1.1", + "wasm-image-optimization": "^2.0.13" }, "devDependencies": { "better-sqlite3": "^13.0.3", @@ -72,7 +72,6 @@ } }, "../llmorbiter": { - "name": "llmorbiter", "version": "0.1.0", "license": "0BSD", "dependencies": { @@ -6436,6 +6435,28 @@ "node": ">=6.0" } }, + "node_modules/wasm-image-optimization": { + "version": "2.0.13", + "resolved": "https://registry.npmjs.org/wasm-image-optimization/-/wasm-image-optimization-2.0.13.tgz", + "integrity": "sha512-MGaFZtdxMHZyaL3VUUcNkN8qccJcy33lcXoUIaegav/y1TiK6c2aALbKAjoe4bXHFI7nDl+5+TG0e9Y8j1/h9Q==", + "license": "MIT", + "dependencies": { + "commander": "15.0.0", + "worker-lib": "2.2.1" + }, + "bin": { + "wasm-image-optimization": "dist/cli.js" + } + }, + "node_modules/wasm-image-optimization/node_modules/commander": { + "version": "15.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-15.0.0.tgz", + "integrity": "sha512-z67u4ZhzCL/Tydu1lJARtEZYWbWaN7oYLHbsuzocr6y4N6WZAagG3RQ4FW61V1/0+jImpj293XfrcYnd1qxtPg==", + "license": "MIT", + "engines": { + "node": ">=22.12.0" + } + }, "node_modules/web-streams-polyfill": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", @@ -6566,6 +6587,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/worker-lib": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/worker-lib/-/worker-lib-2.2.1.tgz", + "integrity": "sha512-CauU4vz3vZUBG3A2CY93GY94KLhbH8JtPKtuvKu2w5G07cHkZ73QkTAJ/lEHDL6arT5WDu1qpU++T0+kGDgSCg==", + "license": "MIT" + }, "node_modules/wrap-ansi": { "version": "10.0.2", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-10.0.2.tgz", diff --git a/package.json b/package.json index 66d8b0e..9197974 100644 --- a/package.json +++ b/package.json @@ -41,10 +41,10 @@ "react": "^19.3.1", "react-canvas-label-editor": "github:sebseb7/react-canvas-label-editor", "selfsigned": "^5.5.0", - "sharp": "^0.35.5", "skelehash-wasm": "^0.1.1", "swagger-ui-dist": "^5.33.3", "vm2": "^3.12.2", + "wasm-image-optimization": "^2.0.13", "zod": "^4.6.7" }, "devDependencies": { diff --git a/scripts/testImageCrop.js b/scripts/testImageCrop.js index d697200..a5c2a0b 100644 --- a/scripts/testImageCrop.js +++ b/scripts/testImageCrop.js @@ -1,73 +1,46 @@ #!/usr/bin/env node -import sharp from 'sharp'; +import { optimizeImage } from 'wasm-image-optimization'; import { cropAndConvertToAvif, crawlAndProcessImage } from '../src/services/imageCrop.js'; async function run() { console.log('--- 1. Testing synthetic images ---'); // Synthetic test 1: White border - const whiteImg = await sharp({ - create: { width: 300, height: 300, channels: 4, background: { r: 255, g: 255, b: 255, alpha: 1 } }, - }) - .composite([ - { - input: Buffer.from(''), - top: 100, - left: 100, - }, - ]) - .png() - .toBuffer(); + const whiteSvg = ` + + + `; + const whiteImg = Buffer.from((await optimizeImage({ image: whiteSvg, format: 'png' })).data); const res1 = await cropAndConvertToAvif(whiteImg); console.log(`White border: 300x300 -> ${res1.width}x${res1.height}, ${res1.size} bytes`); - if (res1.width !== 100 || res1.height !== 100) { - throw new Error(`Expected 100x100, got ${res1.width}x${res1.height}`); + if (res1.width !== 110 || res1.height !== 110) { + throw new Error(`Expected 110x110, got ${res1.width}x${res1.height}`); } // Synthetic test 2: Transparent border - const transImg = await sharp({ - create: { width: 300, height: 300, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } }, - }) - .composite([ - { - input: Buffer.from(''), - top: 110, - left: 90, - }, - ]) - .png() - .toBuffer(); + const transSvg = ` + + `; + const transImg = Buffer.from((await optimizeImage({ image: transSvg, format: 'png' })).data); const res2 = await cropAndConvertToAvif(transImg); console.log(`Transparent border: 300x300 -> ${res2.width}x${res2.height}, ${res2.size} bytes`); - if (res2.width !== 120 || res2.height !== 80) { - throw new Error(`Expected 120x80, got ${res2.width}x${res2.height}`); + if (res2.width !== 130 || res2.height !== 90) { + throw new Error(`Expected 130x90, got ${res2.width}x${res2.height}`); } // Synthetic test 3: Outer transparent, inner white border - const mixedImg = await sharp({ - create: { width: 400, height: 400, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } }, - }) - .composite([ - { - input: Buffer.from(''), - top: 100, - left: 100, - }, - { - input: Buffer.from(''), - top: 170, - left: 150, - }, - ]) - .png() - .toBuffer(); + const mixedSvg = ` + + + `; + const mixedImg = Buffer.from((await optimizeImage({ image: mixedSvg, format: 'png' })).data); const res3 = await cropAndConvertToAvif(mixedImg); console.log(`Mixed border: 400x400 -> ${res3.width}x${res3.height}, ${res3.size} bytes`); - if (res3.width !== 100 || res3.height !== 60) { - throw new Error(`Expected 100x60, got ${res3.width}x${res3.height}`); + if (res3.width !== 110 || res3.height !== 70) { + throw new Error(`Expected 110x70, got ${res3.width}x${res3.height}`); } console.log('\nAll synthetic tests PASSED!'); diff --git a/src/api/routes/serper.js b/src/api/routes/serper.js index a4de4b2..2bdcce6 100644 --- a/src/api/routes/serper.js +++ b/src/api/routes/serper.js @@ -1,4 +1,4 @@ -import sharp from 'sharp'; +import { optimizeImage } from 'wasm-image-optimization'; import * as dbCompanySetting from '../../db/companysetting.js'; import { crawlAndProcessImage } from '../../services/imageCrop.js'; @@ -60,22 +60,29 @@ async function generateWebpThumbnail(img, { signal, preferSource = false } = {}) const arrayBuffer = await response.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); - const thumb = await sharp(buffer) - .rotate() - .resize(200, 200, { - fit: 'inside', - withoutEnlargement: true, - }) - .webp({ quality: 80 }) - .toBuffer({ resolveWithObject: true }); + const meta = await optimizeImage({ image: buffer, format: 'none' }); + const srcW = meta.originalWidth || 0; + const srcH = meta.originalHeight || 0; + const needsResize = srcW > 200 || srcH > 200; + const scale = needsResize ? Math.min(200 / srcW, 200 / srcH) : 1; - const dataUrl = `data:image/webp;base64,${thumb.data.toString('base64')}`; + const thumb = await optimizeImage({ + image: buffer, + width: needsResize ? Math.max(1, Math.round(srcW * scale)) : undefined, + height: needsResize ? Math.max(1, Math.round(srcH * scale)) : undefined, + fit: 'fill', + format: 'webp', + quality: 80, + }); + + const thumbBuffer = Buffer.from(thumb.data); + const dataUrl = `data:image/webp;base64,${thumbBuffer.toString('base64')}`; return { dataUrl, - width: thumb.info.width, - height: thumb.info.height, - size: thumb.info.size, + width: thumb.width, + height: thumb.height, + size: thumbBuffer.length, }; } catch (err) { lastError = err; diff --git a/src/api/routes/vfs.js b/src/api/routes/vfs.js index 38594e7..1131c48 100644 --- a/src/api/routes/vfs.js +++ b/src/api/routes/vfs.js @@ -10,7 +10,7 @@ import { restoreDefaultSkills, restoreDefaultApplets, } from '../../tools/vfs.js'; -import sharp from 'sharp'; +import { optimizeImage } from 'wasm-image-optimization'; // #region Helpers @@ -286,74 +286,79 @@ function buildMetaOverlay({ width, height, size, mime, filename }, outWidth, out * @returns {Promise<{ buffer: Buffer, contentType: string } | null>} null when not rasterisable */ async function renderThumbnail(buffer, { mime, filename }) { - const pipeline = sharp(buffer, { animated: false, failOn: 'none' }); let meta; try { - meta = await pipeline.metadata(); + meta = await optimizeImage({ image: buffer, format: 'none' }); } catch { return null; } - if (!meta?.width || !meta?.height) return null; + const origW = meta?.originalWidth; + const origH = meta?.originalHeight; + if (!origW || !origH) return null; - let scaled = await pipeline - .resize({ - width: THUMB_BOX, - height: THUMB_BOX, - fit: 'inside', - // Downscale only. Upscaling is handled explicitly below, so a 60x40 image ends up at - // 100px (the THUMB_MIN_EDGE rule) rather than being blown up to fill the 250 box. - withoutEnlargement: true, - }) - .png() - .toBuffer(); + // Downscale to fit inside THUMB_BOX without enlargement, then explicitly upscale + // anything whose larger edge is still under THUMB_MIN_EDGE. + const downScale = Math.min(1, THUMB_BOX / origW, THUMB_BOX / origH); + let outWidth = Math.max(1, Math.round(origW * downScale)); + let outHeight = Math.max(1, Math.round(origH * downScale)); - let scaledMeta = await sharp(scaled).metadata(); - - // Upscale small images so they do not render as an unreadable speck. - const largerEdge = Math.max(scaledMeta.width, scaledMeta.height); + const largerEdge = Math.max(outWidth, outHeight); if (largerEdge < THUMB_MIN_EDGE) { const factor = THUMB_MIN_EDGE / largerEdge; - scaled = await sharp(scaled) - .resize({ - width: Math.max(1, Math.round(scaledMeta.width * factor)), - height: Math.max(1, Math.round(scaledMeta.height * factor)), - fit: 'fill', - }) - .png() - .toBuffer(); - scaledMeta = await sharp(scaled).metadata(); + outWidth = Math.max(1, Math.round(outWidth * factor)); + outHeight = Math.max(1, Math.round(outHeight * factor)); } - const outWidth = scaledMeta.width; - const outHeight = scaledMeta.height; - const overlay = buildMetaOverlay( - { - width: meta.width, - height: meta.height, - size: buffer.length, - mime, - filename, - }, - outWidth, + { + width: origW, + height: origH, + size: buffer.length, + mime, + filename, + }, + outWidth, outHeight ); // `null` means the thumbnail is too small to carry a legible metadata band (e.g. an extreme // aspect ratio collapsing to a few pixels tall). Serve the plain scaled image instead. if (!overlay) { - return { - buffer: await sharp(scaled)[THUMB_FORMAT]({ quality: THUMB_QUALITY }).toBuffer(), - contentType: `image/${THUMB_FORMAT}`, - }; + const plain = await optimizeImage({ + image: buffer, + width: outWidth, + height: outHeight, + fit: 'fill', + format: THUMB_FORMAT, + quality: THUMB_QUALITY, + }); + return { + buffer: Buffer.from(plain.data), + contentType: `image/${THUMB_FORMAT}`, + }; } - const composite = await sharp(scaled) - .composite([{ input: overlay, top: 0, left: 0 }]) - [THUMB_FORMAT]({ quality: THUMB_QUALITY }) - .toBuffer(); + + const scaledPng = await optimizeImage({ + image: buffer, + width: outWidth, + height: outHeight, + fit: 'fill', + format: 'png', + }); + const b64 = Buffer.from(scaledPng.data).toString('base64'); + const overlayStr = overlay.toString('utf8'); + const compositeSvg = overlayStr.replace( + /(]*>)/, + `$1` + ); + const composite = await optimizeImage({ + image: compositeSvg, + format: THUMB_FORMAT, + quality: THUMB_QUALITY, + }); return { - buffer: composite, + buffer: Buffer.from(composite.data), contentType: `image/${THUMB_FORMAT}`, }; } diff --git a/src/api/routes/vfsExec.js b/src/api/routes/vfsExec.js index c4cb504..a33becc 100644 --- a/src/api/routes/vfsExec.js +++ b/src/api/routes/vfsExec.js @@ -84,7 +84,7 @@ export function register(app, basePath = '') { * response also carries `columns` (name/type) and `rows` (data row count). Only declared columns are * emitted, in header order; missing values become empty cells. Execution runs in the same isolated worker * sandbox as the `javascript_execute` tool — including authenticated `fetch()` against this API, - * `fs`/`vfs`, `sharp`, ExcelJS, PDFKit and WebCrypto globals. + * `fs`/`vfs`, `optimizeImage`, ExcelJS, PDFKit and WebCrypto globals. * * The path is taken from the URL (like `DELETE /vfs/{*path}`); the run arguments are the JSON body. * Path traversal (`..`), null bytes, and backslashes are rejected with HTTP 400, as is a path that does diff --git a/src/defaultSkills/applet.SKILL b/src/defaultSkills/applet.SKILL index 9dbf291..b77586e 100644 --- a/src/defaultSkills/applet.SKILL +++ b/src/defaultSkills/applet.SKILL @@ -51,7 +51,7 @@ Applets run in an isolated worker thread pool sandbox. - `fs`: Sandboxed filesystem interface for reading/writing local company files. - `ExcelJS`: Pre-injected spreadsheet library. - `PDFDocument`: Pre-injected PDFKit library. - - `sharp`: Pre-injected image manipulation library. + - `optimizeImage`: Pre-injected image manipulation library (`wasm-image-optimization`). - `crypto`: Node / WebCrypto API. - `console.log(...)`: returned in the tool response diff --git a/src/services/companyImport/imageHasher.js b/src/services/companyImport/imageHasher.js index 52bd867..ba07cb7 100644 --- a/src/services/companyImport/imageHasher.js +++ b/src/services/companyImport/imageHasher.js @@ -1,6 +1,6 @@ // MARK: - Imports // #region Imports -import sharp from 'sharp'; +import { optimizeImage } from 'wasm-image-optimization'; import { encodeRGBA } from 'skelehash-wasm'; import { createLogger } from '../../util/logger.js'; // #endregion @@ -23,21 +23,22 @@ export async function computeSkeleHash(buffer) { const base64Data = input.includes(',') ? input.split(',')[1] : input; input = Buffer.from(base64Data, 'base64'); } - const { data, info } = await sharp(input) - .rotate() - .resize({ width: 200, height: 200, fit: 'inside' }) - .ensureAlpha() - .raw() - .toBuffer({ resolveWithObject: true }); + const raw = await optimizeImage({ + image: input, + width: 200, + height: 200, + fit: 'contain', + format: 'raw', + }); - const result = await encodeRGBA(info.width, info.height, data); + const result = await encodeRGBA(raw.width, raw.height, raw.data); if (!result?.base62) { throw new Error('encodeRGBA failed to produce a base62 skelehash'); } return { base62: result.base62, - width: info.width, - height: info.height, + width: raw.width, + height: raw.height, hashLength: result.hash?.length ?? 0, }; } diff --git a/src/services/imageCrop.js b/src/services/imageCrop.js index 9d68588..9801c6d 100644 --- a/src/services/imageCrop.js +++ b/src/services/imageCrop.js @@ -5,18 +5,14 @@ import os from 'os'; const FETCH_TIMEOUT_MS = 15000; const MAX_BYTES = 50 * 1024 * 1024; // 50MB -// The crop/trim/resize/AVIF pipeline runs up to six sequential sharp passes (PNG -// intermediate, trim, extract, resize, AVIF encode). A sharp pipeline is owned by -// whichever thread created it, so running it on the main thread stalls every HTTP -// handler and every SSE client (heartbeats included). The pipeline therefore lives -// in imageCropWorker.js and is reached through the small pool below. +// The crop/trim/resize/AVIF pipeline runs in a worker thread so decoding, +// trimming, and AVIF encoding never stall HTTP handlers or SSE heartbeats. +// The pipeline lives in imageCropWorker.js and is reached through the pool below. const WORKER_PATH = fileURLToPath(new URL('./imageCropWorker.js', import.meta.url)); const WORKER_BOOT_TIMEOUT_MS = 15000; const DEFAULT_JOB_TIMEOUT_MS = 30000; -// libvips already saturates cores via its own per-process thread pool, so each worker -// is mostly IO/decode-bound rather than needing one thread per core. Cap at 3 to stay -// conservative on memory (each in-flight image can be tens of MB) while still covering -// the realistic burst. NOTE: use os.cpus(), not process.availableParallelism() — that +// Cap at 3 workers to stay conservative on memory (each in-flight image can be tens of MB) +// while still covering the realistic burst. NOTE: use os.cpus(), not process.availableParallelism() — that // helper is absent on some supported Node builds and silently collapsed this to 1. const CPU_COUNT = Math.max(1, os.cpus?.()?.length || os.availableParallelism?.() || 2); const POOL_SIZE = Math.max(1, Math.min(3, CPU_COUNT - 1)); @@ -67,8 +63,8 @@ function spawnWorker() { } function onBooted(msg) { - // The worker sends an explicit `{ ready: true }` once its imports (notably - // sharp/libvips) have resolved. Without this handshake there is no first + // The worker sends an explicit `{ ready: true }` once its imports + // have resolved. Without this handshake there is no first // message to await, so a worker that fails to boot would only surface later // as a confusing per-job failure. if (!msg || msg.ready !== true) return; @@ -158,7 +154,7 @@ function runJob(entry, inputBuffer, timeoutMs) { let settled = false; const timer = setTimeout(() => { - // Terminate first, then report. Leaving a worker blocked in libvips would let + // Terminate first, then report. Leaving a worker blocked in WASM would let // it keep burning a core long after the HTTP request has been answered, and // its eventual reply would then arrive against an already-settled job. try { entry.worker.terminate(); } catch { /* already closed */ } @@ -380,8 +376,8 @@ export async function crawlImage(url, { signal } = {}) { * actually trimmed, never enlarging beyond the original), resizes to max 1000px, * and converts to AVIF. * - * Runs the sharp pipeline in a pooled worker thread (see imageCropWorker.js) so the - * multi-pass encode never blocks the main event loop. Errors are re-thrown here with + * Runs the image pipeline in a pooled worker thread (see imageCropWorker.js) so the + * encode never blocks the main event loop. Errors are re-thrown here with * the worker's message intact, so callers that classify on message text (e.g. the * `Failed to crawl image:` prefix check in /serper/image) keep working unchanged. * @@ -400,7 +396,7 @@ export async function cropAndConvertToAvif(inputBuffer, { timeout = DEFAULT_JOB_ // Grow the pool on demand, then queue. enqueueJob/drainQueue hand the job to the // first idle worker, so an idle worker is never leased twice and a saturated pool - // simply waits instead of piling up sharp pipelines. + // simply waits instead of piling up concurrent pipelines. await acquireWorker(); return enqueueJob(inputBuffer, timeout); } diff --git a/src/services/imageCropWorker.js b/src/services/imageCropWorker.js index 7d19897..bdd94f5 100644 --- a/src/services/imageCropWorker.js +++ b/src/services/imageCropWorker.js @@ -1,12 +1,11 @@ 'use strict'; /** - * Worker-thread entry point for the sharp crop/trim/resize/AVIF pipeline. + * Worker-thread entry point for the wasm-image-optimization crop/trim/resize/AVIF pipeline. * - * This pipeline runs up to six sequential sharp passes (PNG intermediate, trim, - * extract, resize, AVIF encode) whose JS glue cost lands on the thread that owns - * the pipeline. Running it here keeps that cost off the main event loop, which - * otherwise stalls HTTP handlers and every SSE client (heartbeats included). + * Running the decode, pixel-trim scan, crop/resize, and AVIF encode in a worker thread + * keeps the cost off the main event loop, which otherwise stalls HTTP handlers and + * every SSE client (heartbeats included). * * Protocol: the parent sends `{ id, buffer }` (an ArrayBuffer, transferred) and * the worker replies `{ id, ok: true, buffer, width, height, size }` (buffer @@ -14,13 +13,13 @@ */ import { parentPort } from 'worker_threads'; -import sharp from 'sharp'; +import { optimizeImage } from 'wasm-image-optimization'; const MAX_EDGE = 1000; const TRIM_THRESHOLD = 10; const RESTORE_MARGIN = 5; // px restored per edge after trimming (up to what was trimmed) const AVIF_QUALITY = 80; -const AVIF_EFFORT = 4; +const AVIF_SPEED = 6; /** * Copies a Buffer into a standalone, transferable ArrayBuffer. @@ -47,90 +46,145 @@ function toTransferable(buffer) { * @returns {Promise<{ buffer: Buffer, width: number, height: number, size: number }>} */ async function cropAndConvertToAvif(inputBuffer) { - // 1. Initial metadata and orientation correction - const baseSharp = sharp(inputBuffer, { failOn: 'none' }).rotate(); - const initialMeta = await baseSharp.metadata(); - const hasAlpha = Boolean(initialMeta.hasAlpha); + // 1. Decode to oriented raw RGBA pixels + const raw = await optimizeImage({ + image: inputBuffer, + format: 'raw', + }); + const origW = raw.width; + const origH = raw.height; + const pixels = raw.data; - // Use PNG intermediate for trimming passes to preserve alpha - const originalBuffer = await baseSharp.png().toBuffer(); - const origMeta = await sharp(originalBuffer).metadata(); + let hasAlpha = false; + for (let i = 3; i < pixels.length; i += 4) { + if (pixels[i] < 255) { + hasAlpha = true; + break; + } + } // Cumulative crop box in original (oriented) pixel coordinates - let box = { left: 0, top: 0, width: origMeta.width, height: origMeta.height }; - let currentBuffer = originalBuffer; + let box = { left: 0, top: 0, width: origW, height: origH }; - const trimPass = async (background) => { - try { - const trimmed = await sharp(currentBuffer) - .trim({ background, threshold: TRIM_THRESHOLD }) - .png() - .toBuffer({ resolveWithObject: true }); - if (trimmed.info.width > 0 && trimmed.info.height > 0) { - currentBuffer = trimmed.data; - box = { - left: box.left - (trimmed.info.trimOffsetLeft ?? 0), - top: box.top - (trimmed.info.trimOffsetTop ?? 0), - width: trimmed.info.width, - height: trimmed.info.height, - }; + const isTransparent = (idx) => pixels[idx + 3] <= TRIM_THRESHOLD; + const isWhite = (idx) => { + const a = pixels[idx + 3]; + if (a < 255 - TRIM_THRESHOLD) return false; + return ( + pixels[idx] >= 255 - TRIM_THRESHOLD && + pixels[idx + 1] >= 255 - TRIM_THRESHOLD && + pixels[idx + 2] >= 255 - TRIM_THRESHOLD + ); + }; + + const trimPass = (isBg) => { + let minX = box.left; + let maxX = box.left + box.width - 1; + let minY = box.top; + let maxY = box.top + box.height - 1; + + while (minY <= maxY) { + let rowBg = true; + for (let x = minX; x <= maxX; x++) { + if (!isBg((minY * origW + x) * 4)) { + rowBg = false; + break; + } } - } catch { - // Keep currentBuffer + if (!rowBg) break; + minY++; + } + + while (maxY >= minY) { + let rowBg = true; + for (let x = minX; x <= maxX; x++) { + if (!isBg((maxY * origW + x) * 4)) { + rowBg = false; + break; + } + } + if (!rowBg) break; + maxY--; + } + + while (minX <= maxX) { + let colBg = true; + for (let y = minY; y <= maxY; y++) { + if (!isBg((y * origW + minX) * 4)) { + colBg = false; + break; + } + } + if (!colBg) break; + minX++; + } + + while (maxX >= minX) { + let colBg = true; + for (let y = minY; y <= maxY; y++) { + if (!isBg((y * origW + maxX) * 4)) { + colBg = false; + break; + } + } + if (!colBg) break; + maxX--; + } + + if (maxX >= minX && maxY >= minY) { + box = { + left: minX, + top: minY, + width: maxX - minX + 1, + height: maxY - minY + 1, + }; } }; // 2. Step A: Trim transparent borders if alpha is present - if (hasAlpha) await trimPass({ r: 0, g: 0, b: 0, alpha: 0 }); + if (hasAlpha) trimPass(isTransparent); // 3. Step B: Trim white borders - await trimPass('#ffffff'); + trimPass(isWhite); // 4. Step C: Trim transparent borders again (handles white border containing transparent padding) - if (hasAlpha) await trimPass({ r: 0, g: 0, b: 0, alpha: 0 }); + if (hasAlpha) trimPass(isTransparent); // 4b. Restore up to RESTORE_MARGIN px per edge from the original, clamped to original bounds const left = Math.max(0, box.left - RESTORE_MARGIN); const top = Math.max(0, box.top - RESTORE_MARGIN); - const right = Math.min(origMeta.width, box.left + box.width + RESTORE_MARGIN); - const bottom = Math.min(origMeta.height, box.top + box.height + RESTORE_MARGIN); - if (left !== box.left || top !== box.top || right - left !== box.width || bottom - top !== box.height) { - try { - const restored = await sharp(originalBuffer) - .extract({ left, top, width: right - left, height: bottom - top }) - .png() - .toBuffer({ resolveWithObject: true }); - currentBuffer = restored.data; - } catch { - // Keep currentBuffer - } - } - const currentMeta = await sharp(currentBuffer).metadata(); + const right = Math.min(origW, box.left + box.width + RESTORE_MARGIN); + const bottom = Math.min(origH, box.top + box.height + RESTORE_MARGIN); + const cropW = right - left; + const cropH = bottom - top; - // 5. Resize to max 1000px without enlargement - let finalPipeline = sharp(currentBuffer); - if (currentMeta.width > MAX_EDGE || currentMeta.height > MAX_EDGE) { - finalPipeline = finalPipeline.resize(MAX_EDGE, MAX_EDGE, { - fit: 'inside', - withoutEnlargement: true, - }); - } + // 5. Resize to max 1000px without enlargement and convert to AVIF + const needsCrop = left > 0 || top > 0 || cropW < origW || cropH < origH; + const needsResize = cropW > MAX_EDGE || cropH > MAX_EDGE; + const scale = needsResize ? Math.min(MAX_EDGE / cropW, MAX_EDGE / cropH) : 1; - // 6. Convert to AVIF - const avifResult = await finalPipeline - .avif({ quality: AVIF_QUALITY, effort: AVIF_EFFORT }) - .toBuffer({ resolveWithObject: true }); + const avifResult = await optimizeImage({ + image: inputBuffer, + crop: needsCrop ? { x: left, y: top, width: cropW, height: cropH } : undefined, + width: needsResize ? Math.max(1, Math.round(cropW * scale)) : undefined, + height: needsResize ? Math.max(1, Math.round(cropH * scale)) : undefined, + fit: 'fill', + format: 'avif', + quality: AVIF_QUALITY, + speed: AVIF_SPEED, + }); + const outBuffer = Buffer.from(avifResult.data); return { - buffer: avifResult.data, - width: avifResult.info.width, - height: avifResult.info.height, - size: avifResult.info.size, + buffer: outBuffer, + width: avifResult.width, + height: avifResult.height, + size: outBuffer.length, }; } // Boot handshake. The parent awaits this before queuing the first job, so a worker -// whose imports (sharp/libvips) fail surfaces as a start-up error rather than as an +// whose imports fail surfaces as a start-up error rather than as an // inexplicable per-job failure. parentPort.postMessage({ ready: true }); diff --git a/src/services/javascriptSandbox.js b/src/services/javascriptSandbox.js index aac2e16..604d720 100644 --- a/src/services/javascriptSandbox.js +++ b/src/services/javascriptSandbox.js @@ -2,7 +2,7 @@ import { Worker } from 'worker_threads'; import { fileURLToPath } from 'url'; import env from '../config/env.js'; -// The sandbox worker pays a large fixed startup cost: importing vm2, sharp, exceljs, pdfkit, +// The sandbox worker pays a large fixed startup cost: importing vm2, wasm-image-optimization, exceljs, pdfkit, // react-canvas-label-editor AND `vfs.js` (which pulls in libdashstore) costs ~700 ms in a fresh // thread, while the snippet itself runs in ~2 ms. ESM modules are never shared between worker // threads, so spawning one thread per applet run re-paid that cost on every request. @@ -17,7 +17,7 @@ const WORKER_BOOT_TIMEOUT_MS = 15000; // Two threads per company rides out the common "run it, then run it again" loop without queueing. const MAX_WORKERS_PER_COMPANY = 2; // ...but the process is shared by every company, so the total stays bounded. Each thread holds -// sharp/libvips plus a NodeVM, so 5 is a deliberate memory ceiling rather than a core count. +// WASM modules plus a NodeVM, so 5 is a deliberate memory ceiling rather than a core count. const MAX_WORKERS_TOTAL = 5; const DEFAULT_BUFFER_LIMIT = 32 * 1024 * 1024; @@ -228,7 +228,7 @@ function runJobOnWorker(entry, job, timeoutMs) { let settled = false; const timer = setTimeout(() => { - // Terminate first, then report. A snippet stuck in a native call (libvips, fetch) + // Terminate first, then report. A snippet stuck in a WASM or I/O call (optimizeImage, fetch) // would otherwise keep burning the thread after the caller has moved on, and its // eventual reply would arrive against an already-settled job. try { entry.worker.terminate(); } catch { /* already closed */ } diff --git a/src/services/labelThumb.js b/src/services/labelThumb.js index 413857e..c675de5 100644 --- a/src/services/labelThumb.js +++ b/src/services/labelThumb.js @@ -1,9 +1,59 @@ -import sharp from 'sharp'; +import zlib from 'node:zlib'; +import { optimizeImage } from 'wasm-image-optimization'; import { renderLabel } from 'react-canvas-label-editor'; const THUMB_W = 150; const THUMB_H = 80; +const CRC_TABLE = (() => { + const table = new Uint32Array(256); + for (let n = 0; n < 256; n++) { + let c = n; + for (let k = 0; k < 8; k++) { + c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + } + table[n] = c; + } + return table; +})(); + +function crc32(buf) { + let c = 0xffffffff; + for (let i = 0; i < buf.length; i++) { + c = CRC_TABLE[(c ^ buf[i]) & 0xff] ^ (c >>> 8); + } + return (c ^ 0xffffffff) >>> 0; +} + +function pngChunk(type, data) { + const b = Buffer.alloc(12 + data.length); + b.writeUInt32BE(data.length, 0); + b.write(type, 4, 4, 'ascii'); + data.copy(b, 8); + b.writeUInt32BE(crc32(b.subarray(4, 8 + data.length)), 8 + data.length); + return b; +} + +function encodeRgbaToPng(w, h, rgba) { + const raw = Buffer.alloc((w * 4 + 1) * h); + for (let y = 0; y < h; y++) { + raw[y * (w * 4 + 1)] = 0; + rgba.copy(raw, y * (w * 4 + 1) + 1, y * w * 4, (y + 1) * w * 4); + } + const idat = zlib.deflateSync(raw, { level: 9 }); + const ihdr = Buffer.alloc(13); + ihdr.writeUInt32BE(w, 0); + ihdr.writeUInt32BE(h, 4); + ihdr[8] = 8; + ihdr[9] = 6; // RGBA + return Buffer.concat([ + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), + pngChunk('IHDR', ihdr), + pngChunk('IDAT', idat), + pngChunk('IEND', Buffer.alloc(0)), + ]); +} + /** * Renders a 150×80 thumbnail for a label template. * @@ -32,17 +82,35 @@ export async function renderThumb({ width, height, objects }) { // 2. Resize to fit within 150×80, preserving aspect ratio. // Transparent background for letterbox padding. - const { data, info } = await sharp(nativePng) - .ensureAlpha() - .resize(THUMB_W, THUMB_H, { - fit: 'contain', - background: { r: 0, g: 0, b: 0, alpha: 0 }, - kernel: sharp.kernel.lanczos3, - }) - .raw() - .toBuffer({ resolveWithObject: true }); + const scaled = await optimizeImage({ + image: nativePng, + width: THUMB_W, + height: THUMB_H, + fit: 'contain', + format: 'raw', + }); - const { width: w, height: h } = info; + const w = THUMB_W; + const h = THUMB_H; + const sw = scaled.width; + const sh = scaled.height; + const offsetX = Math.floor((w - sw) / 2); + const offsetY = Math.floor((h - sh) / 2); + const data = Buffer.alloc(w * h * 4); + for (let y = 0; y < sh; y++) { + const dy = y + offsetY; + if (dy < 0 || dy >= h) continue; + for (let x = 0; x < sw; x++) { + const dx = x + offsetX; + if (dx < 0 || dx >= w) continue; + const srcIdx = (y * sw + x) * 4; + const dstIdx = (dy * w + dx) * 4; + data[dstIdx] = scaled.data[srcIdx]; + data[dstIdx + 1] = scaled.data[srcIdx + 1]; + data[dstIdx + 2] = scaled.data[srcIdx + 2]; + data[dstIdx + 3] = scaled.data[srcIdx + 3]; + } + } // 3. Per-pixel: transparent letterbox remains transparent (alpha=0), // label area is opaque (alpha=255) with 4-bit greyscale (16 levels). @@ -68,10 +136,12 @@ export async function renderThumb({ width, height, objects }) { } } - // 4. Encode as palette PNG with transparency support (16 greys + 1 transparent entry). - const thumb = await sharp(rgba, { raw: { width: w, height: h, channels: 4 } }) - .png({ palette: true, colours: 17, compressionLevel: 9 }) - .toBuffer(); + // 4. Encode as PNG via wasm-image-optimization. + const rawPng = encodeRgbaToPng(w, h, rgba); + const thumb = await optimizeImage({ + image: rawPng, + format: 'png', + }); - return thumb; + return Buffer.from(thumb.data); } diff --git a/src/services/manifestPdf.js b/src/services/manifestPdf.js index 5c1a3a1..4e52dd0 100644 --- a/src/services/manifestPdf.js +++ b/src/services/manifestPdf.js @@ -1,7 +1,7 @@ import PDFDocument from 'pdfkit'; import { createCanvas } from 'canvas'; import JsBarcode from 'jsbarcode'; -import sharp from 'sharp'; +import { optimizeImage } from 'wasm-image-optimization'; import { getStore } from '../db/store.js'; import * as dbOrder from '../db/order.js'; import { emit } from '../api/sse.js'; @@ -127,14 +127,16 @@ export function generateBarcodePng(text, preferredFormat = 'CODE128', options = async function resolveThumbPng(store, googleAuthId, picHash, smallthumbbinary) { if (smallthumbbinary) { try { - return await sharp(smallthumbbinary).png().toBuffer(); + const res = await optimizeImage({ image: smallthumbbinary, format: 'png' }); + return Buffer.from(res.data); } catch { /* fallthrough */ } } if (picHash) { try { const thumb = await store.pic.getThumbByHash(googleAuthId, picHash); if (thumb?.buffer) { - return await sharp(thumb.buffer).png().toBuffer(); + const res = await optimizeImage({ image: thumb.buffer, format: 'png' }); + return Buffer.from(res.data); } } catch { /* ignore */ } } diff --git a/src/tools/analyze-image.js b/src/tools/analyze-image.js index 936b891..faab9a4 100644 --- a/src/tools/analyze-image.js +++ b/src/tools/analyze-image.js @@ -1,6 +1,6 @@ import fs from 'fs'; import path from 'path'; -import sharp from 'sharp'; +import { optimizeImage } from 'wasm-image-optimization'; import { defineTool, ToolError, OpenRouterProvider } from 'libllmorbiter'; import env from '../config/env.js'; import { getImageByHash } from '../db/pic.js'; @@ -130,7 +130,7 @@ function createLabelSvg(name, width, height, fontSize = 11) { */ async function renderGrid(items, config) { const { cols, totalWidth, totalHeight } = config; - const overlays = []; + const elements = []; for (let i = 0; i < items.length; i += 1) { const item = items[i]; @@ -145,36 +145,49 @@ async function renderGrid(items, config) { const maxW = cellW - 2 * LABEL_PAD; const maxH = cellH - LABEL_HEIGHT - 2 * LABEL_PAD; - const resized = await sharp(item.rawBuffer) - .rotate() - .resize(maxW, maxH, { fit: 'inside', withoutEnlargement: true }) - .toBuffer({ resolveWithObject: true }); + const meta = await optimizeImage({ image: item.rawBuffer, format: 'none' }); + const srcW = meta.originalWidth || maxW; + const srcH = meta.originalHeight || maxH; + const scale = Math.min(1, maxW / srcW, maxH / srcH); + const targetW = Math.max(1, Math.round(srcW * scale)); + const targetH = Math.max(1, Math.round(srcH * scale)); - overlays.push({ - input: resized.data, - left: cellX + LABEL_PAD + Math.round((maxW - resized.info.width) / 2), - top: cellY + LABEL_PAD + Math.round((maxH - resized.info.height) / 2), + const resized = await optimizeImage({ + image: item.rawBuffer, + width: scale < 1 ? targetW : undefined, + height: scale < 1 ? targetH : undefined, + fit: 'fill', + format: 'png', }); + const imgLeft = cellX + LABEL_PAD + Math.round((maxW - resized.width) / 2); + const imgTop = cellY + LABEL_PAD + Math.round((maxH - resized.height) / 2); + const b64 = Buffer.from(resized.data).toString('base64'); + elements.push( + `` + ); + const labelW = cellW - 2 * LABEL_PAD; - overlays.push({ - input: createLabelSvg(item.name, labelW, LABEL_HEIGHT, cols === 2 ? 12 : 11), - left: cellX + LABEL_PAD, - top: cellY + cellH - LABEL_HEIGHT - LABEL_PAD, - }); + const labelLeft = cellX + LABEL_PAD; + const labelTop = cellY + cellH - LABEL_HEIGHT - LABEL_PAD; + const labelSvg = createLabelSvg(item.name, labelW, LABEL_HEIGHT, cols === 2 ? 12 : 11) + .toString('utf8') + .replace(/^]*>/, ``); + elements.push(labelSvg); } - return sharp({ - create: { - width: totalWidth, - height: totalHeight, - channels: 4, - background: { r: 20, g: 21, b: 24, alpha: 1 }, - }, - }) - .composite(overlays) - .webp({ quality: WEBP_QUALITY }) - .toBuffer(); + const gridSvg = + `` + + `` + + elements.join('') + + ``; + + const out = await optimizeImage({ + image: gridSvg, + format: 'webp', + quality: WEBP_QUALITY, + }); + return Buffer.from(out.data); } /** @@ -351,11 +364,19 @@ export function createAnalyzeImageTool({ googleAuthId, model }) { const shape = resolveGrid(items.length, grid); let webp; if (items.length === 1 && shape.name === '1x1') { - webp = await sharp(items[0].rawBuffer) - .rotate() - .resize(SINGLE_BOX, SINGLE_BOX, { fit: 'inside', withoutEnlargement: true }) - .webp({ quality: WEBP_QUALITY }) - .toBuffer(); + const meta = await optimizeImage({ image: items[0].rawBuffer, format: 'none' }); + const srcW = meta.originalWidth || SINGLE_BOX; + const srcH = meta.originalHeight || SINGLE_BOX; + const scale = Math.min(1, SINGLE_BOX / srcW, SINGLE_BOX / srcH); + const res = await optimizeImage({ + image: items[0].rawBuffer, + width: scale < 1 ? Math.max(1, Math.round(srcW * scale)) : undefined, + height: scale < 1 ? Math.max(1, Math.round(srcH * scale)) : undefined, + fit: 'fill', + format: 'webp', + quality: WEBP_QUALITY, + }); + webp = Buffer.from(res.data); } else { webp = await renderGrid(items, GRID_CONFIGS[shape.name]); } diff --git a/src/tools/edit-image.js b/src/tools/edit-image.js index 1971303..b733a60 100644 --- a/src/tools/edit-image.js +++ b/src/tools/edit-image.js @@ -1,6 +1,6 @@ import fs from 'fs'; import path from 'path'; -import sharp from 'sharp'; +import { optimizeImage } from 'wasm-image-optimization'; import { defineTool, ToolError } from 'libllmorbiter'; import env from '../config/env.js'; import { getImageByHash } from '../db/pic.js'; @@ -201,26 +201,46 @@ async function loadAttachmentBytes(targetPath, vfsRoot, googleAuthId) { */ async function processAttachmentToDataUrl(rawBuffer, high) { const maxEdge = high ? 1536 : 512; - const metadata = await sharp(rawBuffer).metadata(); - const hasAlpha = Boolean(metadata.hasAlpha); + const raw = await optimizeImage({ image: rawBuffer, format: 'raw' }); + let hasAlpha = false; + for (let i = 3; i < raw.data.length; i += 4) { + if (raw.data[i] < 255) { + hasAlpha = true; + break; + } + } + + const srcW = raw.originalWidth || raw.width || 0; + const srcH = raw.originalHeight || raw.height || 0; + const needsResize = srcW > maxEdge || srcH > maxEdge; + const scale = needsResize ? Math.min(maxEdge / srcW, maxEdge / srcH) : 1; + const width = needsResize ? Math.max(1, Math.round(srcW * scale)) : undefined; + const height = needsResize ? Math.max(1, Math.round(srcH * scale)) : undefined; let mime = 'image/jpeg'; let outputBuffer; if (hasAlpha) { mime = 'image/png'; - outputBuffer = await sharp(rawBuffer) - .rotate() - .resize(maxEdge, maxEdge, { fit: 'inside', withoutEnlargement: true }) - .png({ compressionLevel: high ? 6 : 9 }) - .toBuffer(); + const res = await optimizeImage({ + image: rawBuffer, + width, + height, + fit: 'fill', + format: 'png', + }); + outputBuffer = Buffer.from(res.data); } else { const quality = high ? 90 : 70; - outputBuffer = await sharp(rawBuffer) - .rotate() - .resize(maxEdge, maxEdge, { fit: 'inside', withoutEnlargement: true }) - .jpeg({ quality }) - .toBuffer(); + const res = await optimizeImage({ + image: rawBuffer, + width, + height, + fit: 'fill', + format: 'jpeg', + quality, + }); + outputBuffer = Buffer.from(res.data); } return `data:${mime};base64,${outputBuffer.toString('base64')}`; @@ -447,10 +467,12 @@ export function createEditImageTool({ googleAuthId, model: fallbackModel }) { // ── Convert output to AVIF format ───────────────────────────── let finalBuffer; try { - finalBuffer = await sharp(resultBuffer) - .rotate() - .avif({ quality: AVIF_QUALITY }) - .toBuffer(); + const encoded = await optimizeImage({ + image: resultBuffer, + format: 'avif', + quality: AVIF_QUALITY, + }); + finalBuffer = Buffer.from(encoded.data); } catch (convErr) { throw new ToolError(`edit_image: failed to encode output image to AVIF: ${convErr.message}`); } diff --git a/src/tools/javascript-worker.js b/src/tools/javascript-worker.js index 6572df9..b721fac 100644 --- a/src/tools/javascript-worker.js +++ b/src/tools/javascript-worker.js @@ -2,7 +2,7 @@ import { parentPort, workerData } from 'worker_threads'; import { NodeVM } from 'vm2'; -import sharp from 'sharp'; +import { optimizeImage } from 'wasm-image-optimization'; import path from 'path'; import fsModule from 'fs'; import { validateObjects, validateObject } from 'react-canvas-label-editor'; @@ -149,65 +149,23 @@ function bridgeCrypto() { } const bridgedCrypto = bridgeCrypto(); -// `sharp` writes files DIRECTLY via libvips, bypassing Node's `fs`. Injecting the raw library -// therefore lets a snippet write anywhere the server process can — a relative path lands in -// the server's CWD and an absolute/`../` path escapes the VFS entirely. We wrap the two file -// writing entry points (`toFile` and `toFile` on the constructor, plus `toBuffer` passthrough) -// so output paths are resolved through the VFS containment rules. -// -// `toFile` keeps sharp's real signature/return value; only the path is redirected. -function bridgeSharp(vfs) { - const resolveOut = (p) => { - if (typeof p !== 'string' && !(p && typeof p.toString === 'function')) { - throw new TypeError('sharp.toFile(): output path must be a string'); - } - const diskPath = vfs.resolveWritePath(String(p)); - - // libvips creates missing parent directories itself, so `toFile('a/b/c.png')` would - // silently materialise the whole `a/b/` tree. Node's `fs.writeFileSync` instead fails - // with ENOENT. Enforce the fs semantics here so both write paths behave identically and - // a typo'd path reports an error instead of quietly succeeding. - const dir = path.dirname(diskPath); - if (!fsModule.existsSync(dir)) { - const err = new Error( - `ENOENT: no such file or directory, open '${String(p)}'` - ); - err.code = 'ENOENT'; - err.path = String(p); - err.syscall = 'open'; - throw err; +// `optimizeImage` from `wasm-image-optimization` operates purely in-memory on buffers/SVG strings. +// We wrap it so that a VFS relative path string (when not inline SVG) is resolved through `vfs`. +function bridgeOptimizeImage(vfs) { + return async function sandboxOptimizeImage(params) { + if (!params || typeof params !== 'object') { + throw new TypeError('optimizeImage(): params object is required'); } - - return diskPath; - }; - - // Patch `toFile` on sharp's PROTOTYPE, not on individual instances. - // - // Snippets chain (`sharp(...).png().resize(...).toFile(...)`) and every chained call - // returns a NEW instance, so patching one instance only catches a direct - // `sharp(...).toFile(...)`. Patching the shared prototype catches every derived instance. - const proto = sharp.prototype; - if (proto && typeof proto.toFile === 'function' && !proto.toFile.__vfsPatched) { - const originalToFile = proto.toFile; - const patched = function toFile(file, ...rest) { - return originalToFile.call(this, resolveOut(file), ...rest); + let image = params.image; + if (typeof image === 'string' && !image.trimStart().startsWith('<')) { + image = vfs.readFileSync(image); + } + const res = await optimizeImage({ ...params, image }); + return { + ...res, + data: Buffer.from(res.data), }; - patched.__vfsPatched = true; - proto.toFile = patched; - } - - // Factory passthrough preserving statics + prototype. - const wrapped = function sharpFactory(...args) { - return sharp(...args); }; - - // Preserve statics (sharp.cache, sharp.concurrency, sharp.simd, ...) and prototype. - Object.setPrototypeOf(wrapped, sharp); - for (const key of Object.keys(sharp)) { - try { wrapped[key] = sharp[key]; } catch { /* read-only static */ } - } - - return wrapped; } // ExcelJS reads and writes files via Node's `fs` when using `.readFile()` / `.writeFile()` / streaming. @@ -600,7 +558,7 @@ async function runJob(job) { sandbox: { console: sandboxConsole, fs: vfs, - sharp: bridgeSharp(vfs), + optimizeImage: bridgeOptimizeImage(vfs), ExcelJS: bridgedExcel, exceljs: bridgedExcel, PDFDocument: bridgedPDF, @@ -675,7 +633,7 @@ async function runJob(job) { } } -// Pooled job loop. Imports (sharp, exceljs, pdfkit, libdashstore, …) are paid ONCE for the +// Pooled job loop. Imports (wasm-image-optimization, exceljs, pdfkit, libdashstore, …) are paid ONCE for the // lifetime of the thread; each subsequent job only builds a fresh NodeVM and runs the snippet, // which is the ~2 ms part. Jobs are serialized (one `await` per message) so a pooled thread // never interleaves two applets against the same VFS. diff --git a/src/tools/prompts.js b/src/tools/prompts.js index 8165d38..acc9987 100644 --- a/src/tools/prompts.js +++ b/src/tools/prompts.js @@ -49,7 +49,7 @@ export const TOOL_DESCRIPTIONS = { javascript_execute: 'Execute JS in an isolated sandbox via inline `code` (assign to `module.exports`) or `appletvfsfile` ' + '(.applet.js returning typed CSV per @applet-input-schema, @applet-output-schema, and @applet-description; details in applet.SKILL). ' + - 'Pre-injected globals: fs (use only root-relative paths for all file operations, absolute paths will fail), sharp, ExcelJS, PDFDocument, crypto, and fetch("/path"). No require/import.', + 'Pre-injected globals: fs (use only root-relative paths for all file operations, absolute paths will fail), optimizeImage, ExcelJS, PDFDocument, crypto, and fetch("/path"). No require/import.', find_files: 'List VFS files (root-relative paths). Set skills=true to list root-level ".SKILL" files.', diff --git a/src/tools/search-images.js b/src/tools/search-images.js index 3d05196..a26eb27 100644 --- a/src/tools/search-images.js +++ b/src/tools/search-images.js @@ -1,6 +1,6 @@ import fs from 'fs'; import path from 'path'; -import sharp from 'sharp'; +import { optimizeImage } from 'wasm-image-optimization'; import { defineTool, ToolError } from 'libllmorbiter'; import * as dbCompanySetting from '../db/companysetting.js'; import { prepareVfs, syncVfsToDb } from './vfs.js'; @@ -118,11 +118,20 @@ async function toAvif(img, { signal } = {}) { const input = Buffer.from(await res.arrayBuffer()); if (input.length === 0) continue; - return await sharp(input) - .rotate() // honour EXIF orientation before resizing - .resize(MAX_EDGE, MAX_EDGE, { fit: 'inside', withoutEnlargement: true }) - .avif({ quality: AVIF_QUALITY }) - .toBuffer(); + const meta = await optimizeImage({ image: input, format: 'none' }); + const srcW = meta.originalWidth || 0; + const srcH = meta.originalHeight || 0; + const needsResize = srcW > MAX_EDGE || srcH > MAX_EDGE; + const scale = needsResize ? Math.min(MAX_EDGE / srcW, MAX_EDGE / srcH) : 1; + const out = await optimizeImage({ + image: input, + width: needsResize ? Math.max(1, Math.round(srcW * scale)) : undefined, + height: needsResize ? Math.max(1, Math.round(srcH * scale)) : undefined, + fit: 'fill', + format: 'avif', + quality: AVIF_QUALITY, + }); + return Buffer.from(out.data); } catch { // Try the next candidate URL; a dead imageUrl with a live thumbnail is common. } diff --git a/src/v1/routes/pimage.js b/src/v1/routes/pimage.js index 0cf27a6..9508ac7 100644 --- a/src/v1/routes/pimage.js +++ b/src/v1/routes/pimage.js @@ -1,4 +1,4 @@ -import sharp from 'sharp'; +import { optimizeImage } from 'wasm-image-optimization'; import * as db from '../../db/posterminal.js'; const PIMAGE_DEFAULT_W = 200; @@ -250,42 +250,63 @@ async function servePosImage(req, res, { isCimage = false } = {}) { return res.status(304).end(); } - const imageSharp = sharp(image.buffer); + const meta = await optimizeImage({ image: image.buffer, format: 'none' }); + const srcW = meta.originalWidth || targetW; + const srcH = meta.originalHeight || targetH; + const scale = Math.min(1, targetW / srcW, targetH / srcH); + const scaledW = Math.max(1, Math.round(srcW * scale)); + const scaledH = Math.max(1, Math.round(srcH * scale)); - const resizeOptions = isCimage - ? { - fit: 'contain', - background: '#ffffff', - withoutEnlargement: true, - } - : { - fit: 'inside', - withoutEnlargement: true, - }; + const scaledPng = await optimizeImage({ + image: image.buffer, + width: scale < 1 ? scaledW : undefined, + height: scale < 1 ? scaledH : undefined, + fit: 'fill', + format: 'png', + }); - let pipeline = imageSharp - .resize(targetW, targetH, resizeOptions) - .flatten({ background: '#ffffff' }); + const canvasW = isCimage ? targetW : scaledPng.width; + const canvasH = isCimage ? targetH : scaledPng.height; + const offsetX = Math.round((canvasW - scaledPng.width) / 2); + const offsetY = Math.round((canvasH - scaledPng.height) / 2); + const b64 = Buffer.from(scaledPng.data).toString('base64'); + const flatSvg = + `` + + `` + + `` + + ``; const accept = String(req.headers.accept || ''); const formatParam = typeof req.query.format === 'string' ? req.query.format.toLowerCase() : ''; let contentType = 'image/jpeg'; + let targetFormat = 'jpeg'; + let quality = 85; + let speed = 6; if (formatParam === 'avif' || (!formatParam && accept.includes('image/avif'))) { - pipeline = pipeline.avif({ effort: 2 }); + targetFormat = 'avif'; + speed = 8; contentType = 'image/avif'; } else if (formatParam === 'webp' || (!formatParam && accept.includes('image/webp'))) { - pipeline = pipeline.webp({ quality: 85 }); + targetFormat = 'webp'; + quality = 85; contentType = 'image/webp'; } else if (formatParam === 'png') { - pipeline = pipeline.png(); + targetFormat = 'png'; contentType = 'image/png'; } else { - pipeline = pipeline.jpeg({ quality: 85 }); + targetFormat = 'jpeg'; + quality = 85; contentType = 'image/jpeg'; } - const buffer = await pipeline.toBuffer(); + const encoded = await optimizeImage({ + image: flatSvg, + format: targetFormat, + quality, + speed, + }); + const buffer = Buffer.from(encoded.data); res.setHeader('Content-Type', contentType); res.setHeader('Content-Length', String(buffer.length));