import fs from 'fs'; import path from 'path'; import { optimizeImage } from 'wasm-image-optimization'; import { defineTool, ToolError } from 'libllmorbiter'; import env from '../config/env.js'; import { getImageByHash } from '../db/pic.js'; import { prepareVfs, extractFullHash, isReadOnlyDatabasePath } from './vfs.js'; import { resolveOpenRouterKey, resolveImageModel } from './llm-key.js'; import { TOOL_DESCRIPTIONS, TOOL_PARAMETER_DESCRIPTIONS } from './prompts.js'; /** * edit-image — generate or edit images using the modelslugimage company setting. * * Capabilities: * - Text-to-image: generates an image from scratch when no attachments are passed. * - Image-to-image / editing: transforms, restyles, or composes reference images * supplied via `attachments`, fed in either low or high quality. * * Parameters: * - prompt: text instructions for generation or editing. * - attachments: array of { vfsfilename, high: bool } reference images. * - size: resolution tier — 3 choices ('1K', '2K', '4K'). * - ratio: aspect ratio — 3 choices ('1:1', '4:3', '3:4'). * - vfsoutputfilename: VFS path where the resulting image is saved (always saved as AVIF). */ const DEFAULT_SIZE = '1K'; const DEFAULT_RATIO = '1:1'; const AVIF_QUALITY = 80; const IMAGE_FETCH_TIMEOUT_MS = 10_000; const GENERATE_TIMEOUT_MS = 120_000; const VALID_SIZES = ['1K', '2K', '4K']; const VALID_RATIOS = ['1:1', '4:3', '3:4']; /** * Normalises size parameter into one of the 3 supported tiers ('1K', '2K', '4K'). * @param {string} [val] * @returns {'1K'|'2K'|'4K'} */ function normalizeSize(val) { if (!val) return DEFAULT_SIZE; const clean = String(val).trim().toUpperCase(); if (clean === '1K' || clean === 'SMALL' || clean === '512') return '1K'; if (clean === '2K' || clean === 'MEDIUM' || clean === '1.5K') return '2K'; if (clean === '4K' || clean === 'LARGE') return '4K'; return DEFAULT_SIZE; } /** * Normalises ratio parameter into one of the 3 supported aspect ratios ('1:1', '4:3', '3:4'). * @param {string} [val] * @returns {'1:1'|'4:3'|'3:4'} */ function normalizeRatio(val) { if (!val) return DEFAULT_RATIO; const clean = String(val).trim().toLowerCase(); if (clean === '1:1' || clean === 'square') return '1:1'; if (clean === '4:3' || clean === 'landscape' || clean === 'wide') return '4:3'; if (clean === '3:4' || clean === 'portrait' || clean === 'vertical') return '3:4'; return DEFAULT_RATIO; } /** * Normalises the attachments argument into a clean array of { vfsfilename: string, high: boolean }. * Permissive on argument shapes (array of objects, strings, comma-separated string, JSON array). * * @param {any} raw * @returns {Array<{ vfsfilename: string, high: boolean }>} */ function extractAttachments(raw) { if (!raw) return []; const items = []; const add = (entry) => { if (!entry) return; if (typeof entry === 'string') { const trimmed = entry.trim(); if (!trimmed) return; if (trimmed.startsWith('[') && trimmed.endsWith(']')) { try { const parsed = JSON.parse(trimmed); if (Array.isArray(parsed)) { parsed.forEach(add); return; } } catch { /* treat as plain string */ } } if (trimmed.includes(',')) { trimmed.split(',').map((s) => s.trim()).filter(Boolean).forEach((p) => { items.push({ vfsfilename: p, high: false }); }); return; } items.push({ vfsfilename: trimmed, high: false }); return; } if (typeof entry === 'object') { const file = String(entry.vfsfilename || entry.filename || entry.path || entry.file || '').trim(); if (!file) return; const high = Boolean(entry.high); items.push({ vfsfilename: file, high }); } }; if (Array.isArray(raw)) { raw.forEach(add); } else { add(raw); } return items; } /** * Resolves the OpenRouter Images API URL from configured env.openrouterApiUrl or default. * @returns {string} */ function resolveImagesApiUrl() { if (env.openrouterApiUrl) { try { const u = new URL(env.openrouterApiUrl); u.pathname = u.pathname.replace(/\/chat\/completions\/?$/, '/images'); return u.toString(); } catch { return env.openrouterApiUrl.replace(/\/chat\/completions\/?$/, '/images'); } } return 'https://openrouter.ai/api/v1/images'; } /** * Fetches a database image by its content hash using the database store. * @param {string} hash * @param {number} googleAuthId * @returns {Promise} */ async function fetchDatabaseImage(hash, googleAuthId) { const image = await getImageByHash(googleAuthId, hash); if (!image?.buffer) { throw new Error('Image not found'); } return image.buffer; } /** * Loads image bytes for an attachment path (VFS relative path, DB hash, or HTTP URL). * @param {string} targetPath * @param {string} vfsRoot * @param {number} googleAuthId * @returns {Promise} */ async function loadAttachmentBytes(targetPath, vfsRoot, googleAuthId) { if (/^https?:\/\//i.test(targetPath)) { const res = await fetch(targetPath, { signal: AbortSignal.timeout(IMAGE_FETCH_TIMEOUT_MS) }); if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}`); return Buffer.from(await res.arrayBuffer()); } const hash = extractFullHash(targetPath); if (hash) { const cached = path.join(vfsRoot, 'images', `${hash}.avif`); if (fs.existsSync(cached)) { return fs.readFileSync(cached); } const buf = await fetchDatabaseImage(hash, googleAuthId); try { fs.mkdirSync(path.join(vfsRoot, 'images'), { recursive: true }); fs.writeFileSync(cached, buf); } catch { /* ignore cache write failure */ } return buf; } const clean = targetPath.replace(/^([a-zA-Z]:|\/)+/, '').replace(/^\.\/+/, ''); const abs = path.resolve(vfsRoot, clean); const rel = path.relative(vfsRoot, abs); if (rel.startsWith('..') || path.isAbsolute(rel)) { throw new ToolError(`Access denied: path "${targetPath}" escapes virtual filesystem boundary.`); } if (fs.existsSync(abs) && fs.statSync(abs).isFile()) { return fs.readFileSync(abs); } throw new Error(`File not found at VFS path "${targetPath}".`); } /** * Prepares an attachment buffer for feeding into the model as low or high quality data URL. * * - High quality: up to 1536×1536 max edge, JPEG quality 90 (or PNG with low compression if alpha). * - Low quality: up to 512×512 max edge, JPEG quality 70 (or PNG with high compression if alpha). * * @param {Buffer} rawBuffer * @param {boolean} high * @returns {Promise} data URL */ async function processAttachmentToDataUrl(rawBuffer, high) { const maxEdge = high ? 1536 : 512; const raw = await optimizeImage({ image: rawBuffer, format: 'raw' }); let hasAlpha = false; for (let i = 3; i < raw.data.length; i += 4) { if (raw.data[i] < 255) { hasAlpha = true; break; } } const srcW = raw.originalWidth || raw.width || 0; const srcH = raw.originalHeight || raw.height || 0; const needsResize = srcW > maxEdge || srcH > maxEdge; const scale = needsResize ? Math.min(maxEdge / srcW, maxEdge / srcH) : 1; const width = needsResize ? Math.max(1, Math.round(srcW * scale)) : undefined; const height = needsResize ? Math.max(1, Math.round(srcH * scale)) : undefined; let mime = 'image/jpeg'; let outputBuffer; if (hasAlpha) { mime = 'image/png'; const res = await optimizeImage({ image: rawBuffer, width, height, fit: 'fill', format: 'png', }); outputBuffer = Buffer.from(res.data); } else { const quality = high ? 90 : 70; const res = await optimizeImage({ image: rawBuffer, width, height, fit: 'fill', format: 'jpeg', quality, }); outputBuffer = Buffer.from(res.data); } return `data:${mime};base64,${outputBuffer.toString('base64')}`; } /** * Creates the edit_image tool. * * @param {{ googleAuthId: number, model?: string }} ctx */ export function createEditImageTool({ googleAuthId, model: fallbackModel }) { return defineTool({ name: 'edit_image', description: TOOL_DESCRIPTIONS.edit_image, strict: false, parameters: { type: 'object', additionalProperties: false, required: ['prompt'], properties: { note: { type: ['string', 'null'], description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.note, }, prompt: { type: 'string', description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.prompt, }, attachments: { type: ['array', 'null'], description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.attachments, items: { type: 'object', additionalProperties: false, required: ['vfsfilename'], properties: { vfsfilename: { type: 'string', description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.attachment_vfsfilename, }, high: { type: ['boolean', 'null'], description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.attachment_high, }, }, }, }, size: { type: ['string', 'null'], enum: [...VALID_SIZES, null], description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.size, }, ratio: { type: ['string', 'null'], enum: [...VALID_RATIOS, null], description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.ratio, }, vfsoutputfilename: { type: ['string', 'null'], description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.vfsoutputfilename, }, }, }, invoke: async ({ prompt, attachments, size, ratio, vfsoutputfilename, note } = {}) => { const cleanPrompt = typeof prompt === 'string' ? prompt.trim() : ''; if (!cleanPrompt) { throw new ToolError('edit_image requires a non-empty "prompt" describing the image or edit instructions.'); } const chosenSize = normalizeSize(size); const chosenRatio = normalizeRatio(ratio); const attachmentItems = extractAttachments(attachments); const vfsRoot = await prepareVfs({ googleAuthId }); // ── Prepare attachments (if any) ────────────────────────────── const inputReferences = []; const loadedAttachments = []; for (const att of attachmentItems) { let buf; try { buf = await loadAttachmentBytes(att.vfsfilename, vfsRoot, googleAuthId); } catch (err) { throw new ToolError( `edit_image: failed to load attachment "${att.vfsfilename}": ${err.message}. ` + 'Check available files with find_files or search_images.' ); } if (!buf || buf.length === 0) { throw new ToolError(`edit_image: attachment "${att.vfsfilename}" is empty.`); } const dataUrl = await processAttachmentToDataUrl(buf, att.high); inputReferences.push({ type: 'image_url', image_url: { url: dataUrl }, }); loadedAttachments.push({ vfsfilename: att.vfsfilename, high: att.high, bytes: buf.length, }); } // ── Resolve image model & API key ───────────────────────────── const apiKey = await resolveOpenRouterKey(googleAuthId); if (!apiKey) { throw new ToolError( 'edit_image is unavailable: OpenRouter API key not configured. ' + 'Please configure openrouterApiKey in companysettings or set OPENROUTER_FREE_KEY.' ); } const imageModel = (await resolveImageModel(googleAuthId)) || fallbackModel || 'meta/muse-image'; const apiUrl = resolveImagesApiUrl(); const payload = { model: imageModel, prompt: cleanPrompt, resolution: chosenSize, aspect_ratio: chosenRatio, }; if (inputReferences.length > 0) { payload.input_references = inputReferences; } // ── Call OpenRouter Images API ──────────────────────────────── let response; try { response = await fetch(apiUrl, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${apiKey}`, 'HTTP-Referer': 'https://quixpos.com', 'X-Title': 'QuixPOS', }, body: JSON.stringify(payload), signal: AbortSignal.timeout(GENERATE_TIMEOUT_MS), }); } catch (err) { throw new ToolError(`edit_image: network request to OpenRouter failed: ${err.message}`); } if (!response.ok) { let detail = `${response.status} ${response.statusText}`; try { const errBody = await response.json(); if (errBody?.error?.message) { detail = errBody.error.message; } else if (typeof errBody === 'object') { detail = JSON.stringify(errBody); } } catch { const txt = await response.text().catch(() => ''); if (txt) detail += ` - ${txt.slice(0, 300)}`; } throw new ToolError(`edit_image: OpenRouter Image API error (${detail})`); } const responseData = await response.json(); const item = responseData?.data?.[0]; if (!item) { throw new ToolError('edit_image: OpenRouter returned no image data.'); } let resultBuffer = null; if (item.b64_json) { resultBuffer = Buffer.from(item.b64_json, 'base64'); } else if (item.url) { try { const imgRes = await fetch(item.url, { signal: AbortSignal.timeout(IMAGE_FETCH_TIMEOUT_MS) }); if (!imgRes.ok) throw new Error(`HTTP ${imgRes.status}`); resultBuffer = Buffer.from(await imgRes.arrayBuffer()); } catch (err) { throw new ToolError(`edit_image: failed to download generated image from URL: ${err.message}`); } } if (!resultBuffer || resultBuffer.length === 0) { throw new ToolError('edit_image: received empty image buffer from model.'); } // ── Determine target output path (always .avif) ─────────────── let targetRel = typeof vfsoutputfilename === 'string' && vfsoutputfilename.trim() ? vfsoutputfilename.trim() : ''; if (!targetRel) { const prefix = inputReferences.length > 0 ? 'edited' : 'generated'; targetRel = `images/${prefix}-${Date.now()}.avif`; } let cleanRel = targetRel.replace(/^([a-zA-Z]:|\/)+/, '').replace(/^\.\/+/, ''); if (isReadOnlyDatabasePath(cleanRel)) { throw new ToolError( `Access denied: "${targetRel}" matches read-only database image storage ("image/"). ` + 'Please specify a normal VFS path such as "images/output.avif".' ); } // Always enforce .avif extension const ext = path.extname(cleanRel); if (ext.toLowerCase() !== '.avif') { cleanRel = ext ? `${cleanRel.slice(0, -ext.length)}.avif` : `${cleanRel}.avif`; } if (isReadOnlyDatabasePath(cleanRel)) { throw new ToolError( `Access denied: "${cleanRel}" matches read-only database image storage ("image/"). ` + 'Please specify a normal VFS path such as "images/output.avif".' ); } const absTarget = path.resolve(vfsRoot, cleanRel); const checkRel = path.relative(vfsRoot, absTarget); if (checkRel.startsWith('..') || path.isAbsolute(checkRel)) { throw new ToolError(`Access denied: path "${targetRel}" escapes virtual filesystem boundary.`); } // ── Convert output to AVIF format ───────────────────────────── let finalBuffer; try { const encoded = await optimizeImage({ image: resultBuffer, format: 'avif', quality: AVIF_QUALITY, }); finalBuffer = Buffer.from(encoded.data); } catch (convErr) { throw new ToolError(`edit_image: failed to encode output image to AVIF: ${convErr.message}`); } const mimeType = 'image/avif'; // ── Write to VFS disk and sync to database ──────────────────── fs.mkdirSync(path.dirname(absTarget), { recursive: true }); fs.writeFileSync(absTarget, finalBuffer); const outFinalRel = path.relative(vfsRoot, absTarget).replace(/\\/g, '/'); return { outputFile: outFinalRel, prompt: cleanPrompt, model: imageModel, size: chosenSize, ratio: chosenRatio, attachments: loadedAttachments.map((a) => a.vfsfilename), bytes: finalBuffer.length, mimeType, cost: responseData?.usage?.cost, usage: responseData?.usage, markdown: `![${path.posix.basename(outFinalRel)}](${outFinalRel})`, }; }, }); }