import fs from 'fs'; import path from 'path'; import sharp from 'sharp'; import jwt from 'jsonwebtoken'; import { defineTool, ToolError, OpenRouterProvider } from 'libllmorbiter'; import env from '../config/env.js'; import { prepareVfs } from './vfs.js'; import { resolveOpenRouterKey, resolveImageModel } from './llm-key.js'; /** * analyze_image — lets a vision model actually LOOK at images held in the VFS. * * The agent itself is text-only, so "seeing" an image requires a second, tool-free model * call with the image attached. This tool is that bridge: * * - 1 image → resized to fit 400×400 and sent as a single WebP. * - 2–9 → composited into a labelled grid (2x1/3x1/2x2/3x2/3x3) with the file name * printed under each cell, so one cheap call can compare many candidates. * - Database images are addressed as `image/` and fetched from `/image/:hash`. * * The prompt is assembled from the caller's `prompt` plus a layout summary, so the model * knows which cell is which file. */ const GRID_CONFIGS = { '1x1': { cols: 1, rows: 1, totalWidth: 400, totalHeight: 400 }, '2x1': { cols: 2, rows: 1, totalWidth: 600, totalHeight: 300 }, '3x1': { cols: 3, rows: 1, totalWidth: 800, totalHeight: 300 }, '2x2': { cols: 2, rows: 2, totalWidth: 600, totalHeight: 600 }, '3x2': { cols: 3, rows: 2, totalWidth: 800, totalHeight: 600 }, '3x3': { cols: 3, rows: 3, totalWidth: 800, totalHeight: 800 }, }; const MAX_IMAGES = 9; const SINGLE_BOX = 400; const WEBP_QUALITY = 80; const IMAGE_TIMEOUT_MS = 10_000; const LABEL_HEIGHT = 26; const LABEL_PAD = 5; /** * Picks a grid shape: the caller's choice when valid, otherwise one that fits the count. * @param {number} count * @param {string} [requested] * @returns {{ cols: number, rows: number, name: string }} */ function resolveGrid(count, requested) { if (requested) { const match = String(requested).trim().toLowerCase(); if (GRID_CONFIGS[match]) { const [c, r] = match.split('x').map(Number); return { cols: c, rows: r, name: match }; } } if (count <= 1) return { cols: 1, rows: 1, name: '1x1' }; if (count === 2) return { cols: 2, rows: 1, name: '2x1' }; if (count === 3) return { cols: 3, rows: 1, name: '3x1' }; if (count === 4) return { cols: 2, rows: 2, name: '2x2' }; if (count <= 6) return { cols: 3, rows: 2, name: '3x2' }; return { cols: 3, rows: 3, name: '3x3' }; } /** * Escapes text for safe embedding in the label SVG. * @param {string} unsafe * @returns {string} */ function escapeXml(unsafe) { return String(unsafe).replace(/[<>&'"]/g, (c) => { switch (c) { case '<': return '<'; case '>': return '>'; case '&': return '&'; case "'": return '''; case '"': return '"'; default: return c; } }); } /** * Shortens a label for display, keeping the extension visible so `a.avif` and `b.avif` * never truncate to the same text. * @param {string} name * @param {number} maxChars * @returns {string} */ function formatLabelText(name, maxChars = 28) { if (!name || name.length <= maxChars) return name || ''; const extMatch = name.match(/(\.[a-zA-Z0-9]{3,5})$/); if (extMatch) { const ext = extMatch[1]; const avail = maxChars - ext.length - 1; if (avail > 5) return `${name.slice(0, avail)}…${ext}`; } return `${name.slice(0, maxChars - 1)}…`; } /** * Renders the dark caption bar drawn beneath each grid cell. * @param {string} name * @param {number} width * @param {number} height * @param {number} fontSize * @returns {Buffer} */ function createLabelSvg(name, width, height, fontSize = 11) { const maxChars = Math.max(8, Math.floor(width / (fontSize * 0.65))); const text = escapeXml(formatLabelText(name, maxChars)); const textY = Math.round(height / 2 + fontSize / 3 + 1); return Buffer.from( `` + `` + `${text}` + `` ); } /** * Composites the images into a single labelled grid WebP. * @param {Array<{ name: string, rawBuffer: Buffer }>} items * @param {{ cols: number, rows: number, totalWidth: number, totalHeight: number }} config * @returns {Promise} */ async function renderGrid(items, config) { const { cols, totalWidth, totalHeight } = config; const overlays = []; for (let i = 0; i < items.length; i += 1) { const item = items[i]; const col = i % cols; const row = Math.floor(i / cols); const cellX = Math.round((col * totalWidth) / cols); const cellW = Math.round(((col + 1) * totalWidth) / cols) - cellX; const cellY = Math.round((row * totalHeight) / config.rows); const cellH = Math.round(((row + 1) * totalHeight) / config.rows) - cellY; const maxW = cellW - 2 * LABEL_PAD; const maxH = cellH - LABEL_HEIGHT - 2 * LABEL_PAD; const resized = await sharp(item.rawBuffer) .rotate() .resize(maxW, maxH, { fit: 'inside', withoutEnlargement: true }) .toBuffer({ resolveWithObject: true }); overlays.push({ input: resized.data, left: cellX + LABEL_PAD + Math.round((maxW - resized.info.width) / 2), top: cellY + LABEL_PAD + Math.round((maxH - resized.info.height) / 2), }); const labelW = cellW - 2 * LABEL_PAD; overlays.push({ input: createLabelSvg(item.name, labelW, LABEL_HEIGHT, cols === 2 ? 12 : 11), left: cellX + LABEL_PAD, top: cellY + cellH - LABEL_HEIGHT - LABEL_PAD, }); } return sharp({ create: { width: totalWidth, height: totalHeight, channels: 4, background: { r: 20, g: 21, b: 24, alpha: 1 }, }, }) .composite(overlays) .webp({ quality: WEBP_QUALITY }) .toBuffer(); } /** * Normalises the caller's arguments into a list of `{ path, name }` items. * * Deliberately permissive on input shape (`path` / `paths` / `file` / `images`, string or * array, comma-separated) because models are inconsistent here, while keeping the schema * itself strict. * * @param {object} args * @returns {Array<{ path: string, name: string }>} */ function extractImageInputs(args = {}) { const items = []; const add = (entry) => { if (!entry) return; if (typeof entry === 'string') { const trimmed = entry.trim(); if (!trimmed) return; // Some models send a JSON array as a string. if (trimmed.startsWith('[') && trimmed.endsWith(']')) { try { const parsed = JSON.parse(trimmed); if (Array.isArray(parsed)) { parsed.forEach(add); return; } } catch { /* not JSON, treat as a plain path */ } } if (trimmed.includes(',')) { trimmed.split(',').map((s) => s.trim()).filter(Boolean).forEach((p) => { items.push({ path: p, name: path.posix.basename(p) || p }); }); return; } items.push({ path: trimmed, name: path.posix.basename(trimmed) || trimmed }); return; } if (typeof entry === 'object') { const p = String(entry.path || entry.file || entry.filename || '').trim(); if (!p) return; const name = String(entry.name || entry.title || path.posix.basename(p) || p).trim(); items.push({ path: p, name }); } }; for (const key of ['paths', 'images', 'path', 'file', 'filename']) { const value = args[key]; if (Array.isArray(value)) value.forEach(add); else if (typeof value === 'string') add(value); } if (items.length === 1 && typeof args.name === 'string' && args.name.trim()) { items[0].name = args.name.trim(); } return items; } /** * Fetches a database image by its content hash, authenticated as the calling user. * @param {string} hash 64-char SHA-256 hex * @param {number} googleAuthId * @returns {Promise} */ async function fetchDatabaseImage(hash, googleAuthId) { const url = `http://127.0.0.1:${env.port}/image/${hash}`; const token = jwt.sign({ id: googleAuthId }, env.jwtSecret, { expiresIn: '5m' }); const res = await fetch(url, { headers: { Cookie: `token=${token}` }, signal: AbortSignal.timeout(IMAGE_TIMEOUT_MS), }); if (!res.ok) throw new Error(`${res.status} ${res.statusText}`); return Buffer.from(await res.arrayBuffer()); } /** * Creates the analyze_image tool. * @param {{ googleAuthId: number, model: string }} ctx * `model` is the vision model id from the conversation record; the same model that is * driving the conversation is used for inspection so the agent does not silently switch * vendors mid-task. */ export function createAnalyzeImageTool({ googleAuthId, model }) { return defineTool({ name: 'analyze_image', description: 'Inspect 1–9 images via vision model. Multiple paths render as a labelled comparison grid. ' + 'Paths: VFS-relative ("images/foo.avif") or DB ("image/"). Returns model evaluation.', strict: false, parameters: { type: 'object', additionalProperties: false, required: ['paths'], properties: { paths: { type: 'array', description: '1–9 VFS paths or DB paths ("image/").', items: { type: 'string' }, minItems: 1, maxItems: MAX_IMAGES, }, prompt: { type: 'string', description: 'Inspection criteria or question.', }, grid: { type: 'string', enum: Object.keys(GRID_CONFIGS), description: 'Grid layout. Omit for auto.', }, }, }, invoke: async ({ paths, prompt, grid } = {}) => { const inputs = extractImageInputs({ paths }); if (inputs.length === 0) { throw new ToolError( 'analyze_image requires "paths": an array of 1–9 VFS paths (e.g. ["images/a-1.avif"]) ' + 'or database paths (e.g. ["image/"]). Get candidates from search_images first.' ); } if (inputs.length > MAX_IMAGES) { throw new ToolError(`analyze_image takes at most ${MAX_IMAGES} images per call (received ${inputs.length}). Split into batches.`); } const vfsRoot = await prepareVfs({ googleAuthId }); // ── Load the bytes ──────────────────────────────────────────── const items = []; for (const item of inputs) { // Only a path that actually carries a 64-char content hash is a database image. // A bare "images/…" prefix is just the VFS folder, so it must NOT be treated as // a DB lookup or `images/candidates/x.avif` would be misread as a hash path. const hash = extractFullHash(item.path); let buf = null; if (!hash) { const abs = path.resolve(vfsRoot, item.path.replace(/^([a-zA-Z]:|\/)+/, '').replace(/^\.\/+/, '')); const rel = path.relative(vfsRoot, abs); if (rel.startsWith('..') || path.isAbsolute(rel)) { throw new ToolError(`Access denied: path "${item.path}" escapes virtual filesystem boundary.`); } if (fs.existsSync(abs) && fs.statSync(abs).isFile()) { buf = fs.readFileSync(abs); } } else { const cached = path.join(vfsRoot, 'images', `${hash}.avif`); if (fs.existsSync(cached)) { buf = fs.readFileSync(cached); } else { buf = await fetchDatabaseImage(hash, googleAuthId); } } if (!buf || buf.length === 0) { throw new ToolError( `Image not found or empty: "${item.path}". ` + (hash ? 'Database images must be addressed as "image/" with a 64-char hash.' : 'Store it first (search_images writes to "images/…") or check the path with find_files.') ); } items.push({ ...item, rawBuffer: buf }); } // ── Compose ─────────────────────────────────────────────────── const shape = resolveGrid(items.length, grid); let webp; if (items.length === 1 && shape.name === '1x1') { webp = await sharp(items[0].rawBuffer) .rotate() .resize(SINGLE_BOX, SINGLE_BOX, { fit: 'inside', withoutEnlargement: true }) .webp({ quality: WEBP_QUALITY }) .toBuffer(); } else { webp = await renderGrid(items, GRID_CONFIGS[shape.name]); } // ── Ask the vision model ────────────────────────────────────── const question = typeof prompt === 'string' && prompt.trim() ? prompt.trim() : null; const text = items.length > 1 ? `${question || 'Compare these images, judge visual quality, flag watermarks, text overlays or defects, and say which one is best and why.'}\n\n` + `The grid is ${shape.name}. Each image has its file name printed beneath it:\n` + items.map((it, idx) => { const row = Math.floor(idx / GRID_CONFIGS[shape.name].cols) + 1; const col = (idx % GRID_CONFIGS[shape.name].cols) + 1; return `- Row ${row}, Col ${col}: "${it.name}"`; }).join('\n') : question || 'Describe this image, judge its visual quality, flag any watermark, text overlay or defect, and say whether it is usable as a product or category image.'; const apiKey = await resolveOpenRouterKey(googleAuthId); const imageModel = (await resolveImageModel(googleAuthId)) || model; const provider = new OpenRouterProvider({ apiKey, apiUrl: env.openrouterApiUrl }); const response = await provider.chatCompletions({ model: imageModel, messages: [ { role: 'user', content: [ { type: 'text', text }, { type: 'image_url', image_url: { url: `data:image/webp;base64,${webp.toString('base64')}` }, }, ], }, ], signal: AbortSignal.timeout(120_000), }); const analysis = (response?.content || '').trim(); return { images: items.map((it) => it.path), grid: shape.name, prompt: text, analysis: analysis || '(The vision model returned no text.)', usage: response?.usage ? { input: response.usage.input, output: response.usage.output, cost: response.usage.cost } : undefined, }; }, }); } /** * Extracts a 64-char hex SHA-256 hash from a VFS image path, if present. * Mirrors the sandbox's `extractFullHash` so `image/` resolves identically here. * @param {string} p * @returns {string|null} */ function extractFullHash(p) { if (typeof p !== 'string') return null; const parts = p.trim().replace(/^([a-zA-Z]:|\/)+/, '').split('/'); for (const part of parts) { const seg = part.replace(/\.[a-zA-Z0-9]+$/, ''); if (/^[a-fA-F0-9]{64}$/.test(seg)) return seg.toLowerCase(); } return null; }