505 lines
20 KiB
JavaScript
505 lines
20 KiB
JavaScript
import fs from 'fs';
|
||
import path from 'path';
|
||
import sharp from 'sharp';
|
||
import jwt from 'jsonwebtoken';
|
||
import { defineTool, ToolError } from 'libllmorbiter';
|
||
import env from '../config/env.js';
|
||
import { getImageByHash } from '../db/pic.js';
|
||
import { prepareVfs, syncVfsToDb, extractFullHash, isReadOnlyDatabasePath } from './vfs.js';
|
||
import { resolveOpenRouterKey, resolveImageModel } from './llm-key.js';
|
||
import { TOOL_DESCRIPTIONS, TOOL_PARAMETER_DESCRIPTIONS } from './prompts.js';
|
||
|
||
/**
|
||
* edit-image — generate or edit images using the modelslugimage company setting.
|
||
*
|
||
* Capabilities:
|
||
* - Text-to-image: generates an image from scratch when no attachments are passed.
|
||
* - Image-to-image / editing: transforms, restyles, or composes reference images
|
||
* supplied via `attachments`, fed in either low or high quality.
|
||
*
|
||
* Parameters:
|
||
* - prompt: text instructions for generation or editing.
|
||
* - attachments: array of { vfsfilename, high: bool } reference images.
|
||
* - size: resolution tier — 3 choices ('1K', '2K', '4K').
|
||
* - ratio: aspect ratio — 3 choices ('1:1', '4:3', '3:4').
|
||
* - vfsoutputfilename: VFS path where the resulting image is saved (always saved as AVIF).
|
||
*/
|
||
|
||
const DEFAULT_SIZE = '1K';
|
||
const DEFAULT_RATIO = '1:1';
|
||
const AVIF_QUALITY = 80;
|
||
const IMAGE_FETCH_TIMEOUT_MS = 10_000;
|
||
const GENERATE_TIMEOUT_MS = 120_000;
|
||
|
||
const VALID_SIZES = ['1K', '2K', '4K'];
|
||
const VALID_RATIOS = ['1:1', '4:3', '3:4'];
|
||
|
||
/**
|
||
* Normalises size parameter into one of the 3 supported tiers ('1K', '2K', '4K').
|
||
* @param {string} [val]
|
||
* @returns {'1K'|'2K'|'4K'}
|
||
*/
|
||
function normalizeSize(val) {
|
||
if (!val) return DEFAULT_SIZE;
|
||
const clean = String(val).trim().toUpperCase();
|
||
if (clean === '1K' || clean === 'SMALL' || clean === '512') return '1K';
|
||
if (clean === '2K' || clean === 'MEDIUM' || clean === '1.5K') return '2K';
|
||
if (clean === '4K' || clean === 'LARGE') return '4K';
|
||
return DEFAULT_SIZE;
|
||
}
|
||
|
||
/**
|
||
* Normalises ratio parameter into one of the 3 supported aspect ratios ('1:1', '4:3', '3:4').
|
||
* @param {string} [val]
|
||
* @returns {'1:1'|'4:3'|'3:4'}
|
||
*/
|
||
function normalizeRatio(val) {
|
||
if (!val) return DEFAULT_RATIO;
|
||
const clean = String(val).trim().toLowerCase();
|
||
if (clean === '1:1' || clean === 'square') return '1:1';
|
||
if (clean === '4:3' || clean === 'landscape' || clean === 'wide') return '4:3';
|
||
if (clean === '3:4' || clean === 'portrait' || clean === 'vertical') return '3:4';
|
||
return DEFAULT_RATIO;
|
||
}
|
||
|
||
/**
|
||
* Normalises the attachments argument into a clean array of { vfsfilename: string, high: boolean }.
|
||
* Permissive on argument shapes (array of objects, strings, comma-separated string, JSON array).
|
||
*
|
||
* @param {any} raw
|
||
* @returns {Array<{ vfsfilename: string, high: boolean }>}
|
||
*/
|
||
function extractAttachments(raw) {
|
||
if (!raw) return [];
|
||
const items = [];
|
||
|
||
const add = (entry) => {
|
||
if (!entry) return;
|
||
if (typeof entry === 'string') {
|
||
const trimmed = entry.trim();
|
||
if (!trimmed) return;
|
||
|
||
if (trimmed.startsWith('[') && trimmed.endsWith(']')) {
|
||
try {
|
||
const parsed = JSON.parse(trimmed);
|
||
if (Array.isArray(parsed)) {
|
||
parsed.forEach(add);
|
||
return;
|
||
}
|
||
} catch { /* treat as plain string */ }
|
||
}
|
||
|
||
if (trimmed.includes(',')) {
|
||
trimmed.split(',').map((s) => s.trim()).filter(Boolean).forEach((p) => {
|
||
items.push({ vfsfilename: p, high: false });
|
||
});
|
||
return;
|
||
}
|
||
|
||
items.push({ vfsfilename: trimmed, high: false });
|
||
return;
|
||
}
|
||
|
||
if (typeof entry === 'object') {
|
||
const file = String(entry.vfsfilename || entry.filename || entry.path || entry.file || '').trim();
|
||
if (!file) return;
|
||
const high = Boolean(entry.high);
|
||
items.push({ vfsfilename: file, high });
|
||
}
|
||
};
|
||
|
||
if (Array.isArray(raw)) {
|
||
raw.forEach(add);
|
||
} else {
|
||
add(raw);
|
||
}
|
||
|
||
return items;
|
||
}
|
||
|
||
/**
|
||
* Resolves the OpenRouter Images API URL from configured env.openrouterApiUrl or default.
|
||
* @returns {string}
|
||
*/
|
||
function resolveImagesApiUrl() {
|
||
if (env.openrouterApiUrl) {
|
||
try {
|
||
const u = new URL(env.openrouterApiUrl);
|
||
u.pathname = u.pathname.replace(/\/chat\/completions\/?$/, '/images');
|
||
return u.toString();
|
||
} catch {
|
||
return env.openrouterApiUrl.replace(/\/chat\/completions\/?$/, '/images');
|
||
}
|
||
}
|
||
return 'https://openrouter.ai/api/v1/images';
|
||
}
|
||
|
||
/**
|
||
* Fetches a database image by its 8-char content hash using store or authenticated local endpoint.
|
||
* @param {string} hash
|
||
* @param {number} googleAuthId
|
||
* @returns {Promise<Buffer>}
|
||
*/
|
||
async function fetchDatabaseImage(hash, googleAuthId) {
|
||
try {
|
||
const image = await getImageByHash(googleAuthId, hash);
|
||
if (image?.buffer) {
|
||
return image.buffer;
|
||
}
|
||
} catch (dbErr) {
|
||
try {
|
||
const url = `http://127.0.0.1:${env.port}/api/image/${hash}`;
|
||
const token = jwt.sign({ id: googleAuthId }, env.jwtSecret, { expiresIn: '5m' });
|
||
const res = await fetch(url, {
|
||
headers: { Cookie: `token=${token}` },
|
||
signal: AbortSignal.timeout(IMAGE_FETCH_TIMEOUT_MS),
|
||
});
|
||
if (res.ok) {
|
||
return Buffer.from(await res.arrayBuffer());
|
||
}
|
||
} catch {
|
||
// fallback error ignored
|
||
}
|
||
throw dbErr;
|
||
}
|
||
throw new Error('Image not found');
|
||
}
|
||
|
||
/**
|
||
* Loads image bytes for an attachment path (VFS relative path, DB hash, or HTTP URL).
|
||
* @param {string} targetPath
|
||
* @param {string} vfsRoot
|
||
* @param {number} googleAuthId
|
||
* @returns {Promise<Buffer>}
|
||
*/
|
||
async function loadAttachmentBytes(targetPath, vfsRoot, googleAuthId) {
|
||
if (/^https?:\/\//i.test(targetPath)) {
|
||
const res = await fetch(targetPath, { signal: AbortSignal.timeout(IMAGE_FETCH_TIMEOUT_MS) });
|
||
if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}`);
|
||
return Buffer.from(await res.arrayBuffer());
|
||
}
|
||
|
||
const hash = extractFullHash(targetPath);
|
||
if (hash) {
|
||
const cached = path.join(vfsRoot, 'images', `${hash}.avif`);
|
||
if (fs.existsSync(cached)) {
|
||
return fs.readFileSync(cached);
|
||
}
|
||
const buf = await fetchDatabaseImage(hash, googleAuthId);
|
||
try {
|
||
fs.mkdirSync(path.join(vfsRoot, 'images'), { recursive: true });
|
||
fs.writeFileSync(cached, buf);
|
||
} catch { /* ignore cache write failure */ }
|
||
return buf;
|
||
}
|
||
|
||
const clean = targetPath.replace(/^([a-zA-Z]:|\/)+/, '').replace(/^\.\/+/, '');
|
||
const abs = path.resolve(vfsRoot, clean);
|
||
const rel = path.relative(vfsRoot, abs);
|
||
if (rel.startsWith('..') || path.isAbsolute(rel)) {
|
||
throw new ToolError(`Access denied: path "${targetPath}" escapes virtual filesystem boundary.`);
|
||
}
|
||
|
||
if (fs.existsSync(abs) && fs.statSync(abs).isFile()) {
|
||
return fs.readFileSync(abs);
|
||
}
|
||
|
||
throw new Error(`File not found at VFS path "${targetPath}".`);
|
||
}
|
||
|
||
/**
|
||
* Prepares an attachment buffer for feeding into the model as low or high quality data URL.
|
||
*
|
||
* - High quality: up to 1536×1536 max edge, JPEG quality 90 (or PNG with low compression if alpha).
|
||
* - Low quality: up to 512×512 max edge, JPEG quality 70 (or PNG with high compression if alpha).
|
||
*
|
||
* @param {Buffer} rawBuffer
|
||
* @param {boolean} high
|
||
* @returns {Promise<string>} data URL
|
||
*/
|
||
async function processAttachmentToDataUrl(rawBuffer, high) {
|
||
const maxEdge = high ? 1536 : 512;
|
||
const metadata = await sharp(rawBuffer).metadata();
|
||
const hasAlpha = Boolean(metadata.hasAlpha);
|
||
|
||
let mime = 'image/jpeg';
|
||
let outputBuffer;
|
||
|
||
if (hasAlpha) {
|
||
mime = 'image/png';
|
||
outputBuffer = await sharp(rawBuffer)
|
||
.rotate()
|
||
.resize(maxEdge, maxEdge, { fit: 'inside', withoutEnlargement: true })
|
||
.png({ compressionLevel: high ? 6 : 9 })
|
||
.toBuffer();
|
||
} else {
|
||
const quality = high ? 90 : 70;
|
||
outputBuffer = await sharp(rawBuffer)
|
||
.rotate()
|
||
.resize(maxEdge, maxEdge, { fit: 'inside', withoutEnlargement: true })
|
||
.jpeg({ quality })
|
||
.toBuffer();
|
||
}
|
||
|
||
return `data:${mime};base64,${outputBuffer.toString('base64')}`;
|
||
}
|
||
|
||
/**
|
||
* Creates the edit_image tool.
|
||
*
|
||
* @param {{ googleAuthId: number, model?: string }} ctx
|
||
*/
|
||
export function createEditImageTool({ googleAuthId, model: fallbackModel }) {
|
||
return defineTool({
|
||
name: 'edit_image',
|
||
description: TOOL_DESCRIPTIONS.edit_image,
|
||
strict: false,
|
||
parameters: {
|
||
type: 'object',
|
||
additionalProperties: false,
|
||
required: ['prompt'],
|
||
properties: {
|
||
note: {
|
||
type: ['string', 'null'],
|
||
description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.note,
|
||
},
|
||
prompt: {
|
||
type: 'string',
|
||
description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.prompt,
|
||
},
|
||
attachments: {
|
||
type: ['array', 'null'],
|
||
description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.attachments,
|
||
items: {
|
||
type: 'object',
|
||
additionalProperties: false,
|
||
required: ['vfsfilename'],
|
||
properties: {
|
||
vfsfilename: {
|
||
type: 'string',
|
||
description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.attachment_vfsfilename,
|
||
},
|
||
high: {
|
||
type: ['boolean', 'null'],
|
||
description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.attachment_high,
|
||
},
|
||
},
|
||
},
|
||
},
|
||
size: {
|
||
type: ['string', 'null'],
|
||
enum: [...VALID_SIZES, null],
|
||
description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.size,
|
||
},
|
||
ratio: {
|
||
type: ['string', 'null'],
|
||
enum: [...VALID_RATIOS, null],
|
||
description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.ratio,
|
||
},
|
||
vfsoutputfilename: {
|
||
type: ['string', 'null'],
|
||
description: TOOL_PARAMETER_DESCRIPTIONS.edit_image.vfsoutputfilename,
|
||
},
|
||
},
|
||
},
|
||
invoke: async ({ prompt, attachments, size, ratio, vfsoutputfilename, note } = {}) => {
|
||
const cleanPrompt = typeof prompt === 'string' ? prompt.trim() : '';
|
||
if (!cleanPrompt) {
|
||
throw new ToolError('edit_image requires a non-empty "prompt" describing the image or edit instructions.');
|
||
}
|
||
|
||
const chosenSize = normalizeSize(size);
|
||
const chosenRatio = normalizeRatio(ratio);
|
||
const attachmentItems = extractAttachments(attachments);
|
||
|
||
const vfsRoot = await prepareVfs({ googleAuthId });
|
||
|
||
// ── Prepare attachments (if any) ──────────────────────────────
|
||
const inputReferences = [];
|
||
const loadedAttachments = [];
|
||
|
||
for (const att of attachmentItems) {
|
||
let buf;
|
||
try {
|
||
buf = await loadAttachmentBytes(att.vfsfilename, vfsRoot, googleAuthId);
|
||
} catch (err) {
|
||
throw new ToolError(
|
||
`edit_image: failed to load attachment "${att.vfsfilename}": ${err.message}. ` +
|
||
'Check available files with find_files or search_images.'
|
||
);
|
||
}
|
||
|
||
if (!buf || buf.length === 0) {
|
||
throw new ToolError(`edit_image: attachment "${att.vfsfilename}" is empty.`);
|
||
}
|
||
|
||
const dataUrl = await processAttachmentToDataUrl(buf, att.high);
|
||
inputReferences.push({
|
||
type: 'image_url',
|
||
image_url: { url: dataUrl },
|
||
});
|
||
loadedAttachments.push({
|
||
vfsfilename: att.vfsfilename,
|
||
high: att.high,
|
||
bytes: buf.length,
|
||
});
|
||
}
|
||
|
||
// ── Resolve image model & API key ─────────────────────────────
|
||
const apiKey = await resolveOpenRouterKey(googleAuthId);
|
||
if (!apiKey) {
|
||
throw new ToolError(
|
||
'edit_image is unavailable: OpenRouter API key not configured. ' +
|
||
'Please configure openrouterApiKey in companysettings or set OPENROUTER_FREE_KEY.'
|
||
);
|
||
}
|
||
|
||
const imageModel = (await resolveImageModel(googleAuthId)) || fallbackModel || 'meta/muse-image';
|
||
const apiUrl = resolveImagesApiUrl();
|
||
|
||
const payload = {
|
||
model: imageModel,
|
||
prompt: cleanPrompt,
|
||
resolution: chosenSize,
|
||
aspect_ratio: chosenRatio,
|
||
};
|
||
|
||
if (inputReferences.length > 0) {
|
||
payload.input_references = inputReferences;
|
||
}
|
||
|
||
// ── Call OpenRouter Images API ────────────────────────────────
|
||
let response;
|
||
try {
|
||
response = await fetch(apiUrl, {
|
||
method: 'POST',
|
||
headers: {
|
||
'Content-Type': 'application/json',
|
||
'Authorization': `Bearer ${apiKey}`,
|
||
'HTTP-Referer': 'https://quixpos.com',
|
||
'X-Title': 'QuixPOS',
|
||
},
|
||
body: JSON.stringify(payload),
|
||
signal: AbortSignal.timeout(GENERATE_TIMEOUT_MS),
|
||
});
|
||
} catch (err) {
|
||
throw new ToolError(`edit_image: network request to OpenRouter failed: ${err.message}`);
|
||
}
|
||
|
||
if (!response.ok) {
|
||
let detail = `${response.status} ${response.statusText}`;
|
||
try {
|
||
const errBody = await response.json();
|
||
if (errBody?.error?.message) {
|
||
detail = errBody.error.message;
|
||
} else if (typeof errBody === 'object') {
|
||
detail = JSON.stringify(errBody);
|
||
}
|
||
} catch {
|
||
const txt = await response.text().catch(() => '');
|
||
if (txt) detail += ` - ${txt.slice(0, 300)}`;
|
||
}
|
||
throw new ToolError(`edit_image: OpenRouter Image API error (${detail})`);
|
||
}
|
||
|
||
const responseData = await response.json();
|
||
const item = responseData?.data?.[0];
|
||
if (!item) {
|
||
throw new ToolError('edit_image: OpenRouter returned no image data.');
|
||
}
|
||
|
||
let resultBuffer = null;
|
||
if (item.b64_json) {
|
||
resultBuffer = Buffer.from(item.b64_json, 'base64');
|
||
} else if (item.url) {
|
||
try {
|
||
const imgRes = await fetch(item.url, { signal: AbortSignal.timeout(IMAGE_FETCH_TIMEOUT_MS) });
|
||
if (!imgRes.ok) throw new Error(`HTTP ${imgRes.status}`);
|
||
resultBuffer = Buffer.from(await imgRes.arrayBuffer());
|
||
} catch (err) {
|
||
throw new ToolError(`edit_image: failed to download generated image from URL: ${err.message}`);
|
||
}
|
||
}
|
||
|
||
if (!resultBuffer || resultBuffer.length === 0) {
|
||
throw new ToolError('edit_image: received empty image buffer from model.');
|
||
}
|
||
|
||
// ── Determine target output path (always .avif) ───────────────
|
||
let targetRel = typeof vfsoutputfilename === 'string' && vfsoutputfilename.trim()
|
||
? vfsoutputfilename.trim()
|
||
: '';
|
||
|
||
if (!targetRel) {
|
||
const prefix = inputReferences.length > 0 ? 'edited' : 'generated';
|
||
targetRel = `images/${prefix}-${Date.now()}.avif`;
|
||
}
|
||
|
||
let cleanRel = targetRel.replace(/^([a-zA-Z]:|\/)+/, '').replace(/^\.\/+/, '');
|
||
if (isReadOnlyDatabasePath(cleanRel)) {
|
||
throw new ToolError(
|
||
`Access denied: "${targetRel}" matches read-only database image storage ("image/<hash>"). ` +
|
||
'Please specify a normal VFS path such as "images/output.avif".'
|
||
);
|
||
}
|
||
|
||
// Always enforce .avif extension
|
||
const ext = path.extname(cleanRel);
|
||
if (ext.toLowerCase() !== '.avif') {
|
||
cleanRel = ext ? `${cleanRel.slice(0, -ext.length)}.avif` : `${cleanRel}.avif`;
|
||
}
|
||
|
||
if (isReadOnlyDatabasePath(cleanRel)) {
|
||
throw new ToolError(
|
||
`Access denied: "${cleanRel}" matches read-only database image storage ("image/<hash>"). ` +
|
||
'Please specify a normal VFS path such as "images/output.avif".'
|
||
);
|
||
}
|
||
|
||
const absTarget = path.resolve(vfsRoot, cleanRel);
|
||
const checkRel = path.relative(vfsRoot, absTarget);
|
||
if (checkRel.startsWith('..') || path.isAbsolute(checkRel)) {
|
||
throw new ToolError(`Access denied: path "${targetRel}" escapes virtual filesystem boundary.`);
|
||
}
|
||
|
||
// ── Convert output to AVIF format ─────────────────────────────
|
||
let finalBuffer;
|
||
try {
|
||
finalBuffer = await sharp(resultBuffer)
|
||
.rotate()
|
||
.avif({ quality: AVIF_QUALITY })
|
||
.toBuffer();
|
||
} catch (convErr) {
|
||
throw new ToolError(`edit_image: failed to encode output image to AVIF: ${convErr.message}`);
|
||
}
|
||
const mimeType = 'image/avif';
|
||
|
||
// ── Write to VFS disk and sync to database ────────────────────
|
||
fs.mkdirSync(path.dirname(absTarget), { recursive: true });
|
||
fs.writeFileSync(absTarget, finalBuffer);
|
||
|
||
try {
|
||
await syncVfsToDb({ googleAuthId, vfsRoot });
|
||
} catch (syncErr) {
|
||
console.warn('[edit_image] warning: failed to sync output image to modelvfs db:', syncErr.message);
|
||
}
|
||
|
||
const outFinalRel = path.relative(vfsRoot, absTarget).replace(/\\/g, '/');
|
||
|
||
return {
|
||
outputFile: outFinalRel,
|
||
prompt: cleanPrompt,
|
||
model: imageModel,
|
||
size: chosenSize,
|
||
ratio: chosenRatio,
|
||
attachments: loadedAttachments.map((a) => a.vfsfilename),
|
||
bytes: finalBuffer.length,
|
||
mimeType,
|
||
cost: responseData?.usage?.cost,
|
||
usage: responseData?.usage,
|
||
markdown: ``,
|
||
};
|
||
},
|
||
});
|
||
}
|