u
This commit is contained in:
seb
419
src/tools/analyze-image.js
Normal file
419
src/tools/analyze-image.js
Normal file
@@ -0,0 +1,419 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import sharp from 'sharp';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { defineTool, ToolError, OpenRouterProvider } from 'libllmorbiter';
|
||||
import env from '../config/env.js';
|
||||
import { prepareVfs } from './vfs.js';
|
||||
import { resolveOpenRouterKey } from './llm-key.js';
|
||||
|
||||
/**
|
||||
* analyze_image — lets a vision model actually LOOK at images held in the VFS.
|
||||
*
|
||||
* The agent itself is text-only, so "seeing" an image requires a second, tool-free model
|
||||
* call with the image attached. This tool is that bridge:
|
||||
*
|
||||
* - 1 image → resized to fit 400×400 and sent as a single WebP.
|
||||
* - 2–9 → composited into a labelled grid (2x1/3x1/2x2/3x2/3x3) with the file name
|
||||
* printed under each cell, so one cheap call can compare many candidates.
|
||||
* - Database images are addressed as `image/<fullHash>` and fetched from `/image/:hash`.
|
||||
*
|
||||
* The prompt is assembled from the caller's `prompt` plus a layout summary, so the model
|
||||
* knows which cell is which file.
|
||||
*/
|
||||
|
||||
const GRID_CONFIGS = {
|
||||
'1x1': { cols: 1, rows: 1, totalWidth: 400, totalHeight: 400 },
|
||||
'2x1': { cols: 2, rows: 1, totalWidth: 600, totalHeight: 300 },
|
||||
'3x1': { cols: 3, rows: 1, totalWidth: 800, totalHeight: 300 },
|
||||
'2x2': { cols: 2, rows: 2, totalWidth: 600, totalHeight: 600 },
|
||||
'3x2': { cols: 3, rows: 2, totalWidth: 800, totalHeight: 600 },
|
||||
'3x3': { cols: 3, rows: 3, totalWidth: 800, totalHeight: 800 },
|
||||
};
|
||||
|
||||
const MAX_IMAGES = 9;
|
||||
const SINGLE_BOX = 400;
|
||||
const WEBP_QUALITY = 80;
|
||||
const IMAGE_TIMEOUT_MS = 10_000;
|
||||
const LABEL_HEIGHT = 26;
|
||||
const LABEL_PAD = 5;
|
||||
|
||||
/**
|
||||
* Picks a grid shape: the caller's choice when valid, otherwise one that fits the count.
|
||||
* @param {number} count
|
||||
* @param {string} [requested]
|
||||
* @returns {{ cols: number, rows: number, name: string }}
|
||||
*/
|
||||
function resolveGrid(count, requested) {
|
||||
if (requested) {
|
||||
const match = String(requested).trim().toLowerCase();
|
||||
if (GRID_CONFIGS[match]) {
|
||||
const [c, r] = match.split('x').map(Number);
|
||||
return { cols: c, rows: r, name: match };
|
||||
}
|
||||
}
|
||||
if (count <= 1) return { cols: 1, rows: 1, name: '1x1' };
|
||||
if (count === 2) return { cols: 2, rows: 1, name: '2x1' };
|
||||
if (count === 3) return { cols: 3, rows: 1, name: '3x1' };
|
||||
if (count === 4) return { cols: 2, rows: 2, name: '2x2' };
|
||||
if (count <= 6) return { cols: 3, rows: 2, name: '3x2' };
|
||||
return { cols: 3, rows: 3, name: '3x3' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Escapes text for safe embedding in the label SVG.
|
||||
* @param {string} unsafe
|
||||
* @returns {string}
|
||||
*/
|
||||
function escapeXml(unsafe) {
|
||||
return String(unsafe).replace(/[<>&'"]/g, (c) => {
|
||||
switch (c) {
|
||||
case '<': return '<';
|
||||
case '>': return '>';
|
||||
case '&': return '&';
|
||||
case "'": return ''';
|
||||
case '"': return '"';
|
||||
default: return c;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Shortens a label for display, keeping the extension visible so `a.avif` and `b.avif`
|
||||
* never truncate to the same text.
|
||||
* @param {string} name
|
||||
* @param {number} maxChars
|
||||
* @returns {string}
|
||||
*/
|
||||
function formatLabelText(name, maxChars = 28) {
|
||||
if (!name || name.length <= maxChars) return name || '';
|
||||
const extMatch = name.match(/(\.[a-zA-Z0-9]{3,5})$/);
|
||||
if (extMatch) {
|
||||
const ext = extMatch[1];
|
||||
const avail = maxChars - ext.length - 1;
|
||||
if (avail > 5) return `${name.slice(0, avail)}…${ext}`;
|
||||
}
|
||||
return `${name.slice(0, maxChars - 1)}…`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders the dark caption bar drawn beneath each grid cell.
|
||||
* @param {string} name
|
||||
* @param {number} width
|
||||
* @param {number} height
|
||||
* @param {number} fontSize
|
||||
* @returns {Buffer}
|
||||
*/
|
||||
function createLabelSvg(name, width, height, fontSize = 11) {
|
||||
const maxChars = Math.max(8, Math.floor(width / (fontSize * 0.65)));
|
||||
const text = escapeXml(formatLabelText(name, maxChars));
|
||||
const textY = Math.round(height / 2 + fontSize / 3 + 1);
|
||||
return Buffer.from(
|
||||
`<svg width="${width}" height="${height}" xmlns="http://www.w3.org/2000/svg">` +
|
||||
`<rect width="100%" height="100%" fill="#1e2025" rx="3" stroke="#333842" stroke-width="1"/>` +
|
||||
`<text x="${Math.round(width / 2)}" y="${textY}" font-family="system-ui,-apple-system,sans-serif" ` +
|
||||
`font-size="${fontSize}" font-weight="600" fill="#f1f5f9" text-anchor="middle">${text}</text>` +
|
||||
`</svg>`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Composites the images into a single labelled grid WebP.
|
||||
* @param {Array<{ name: string, rawBuffer: Buffer }>} items
|
||||
* @param {{ cols: number, rows: number, totalWidth: number, totalHeight: number }} config
|
||||
* @returns {Promise<Buffer>}
|
||||
*/
|
||||
async function renderGrid(items, config) {
|
||||
const { cols, totalWidth, totalHeight } = config;
|
||||
const overlays = [];
|
||||
|
||||
for (let i = 0; i < items.length; i += 1) {
|
||||
const item = items[i];
|
||||
const col = i % cols;
|
||||
const row = Math.floor(i / cols);
|
||||
|
||||
const cellX = Math.round((col * totalWidth) / cols);
|
||||
const cellW = Math.round(((col + 1) * totalWidth) / cols) - cellX;
|
||||
const cellY = Math.round((row * totalHeight) / config.rows);
|
||||
const cellH = Math.round(((row + 1) * totalHeight) / config.rows) - cellY;
|
||||
|
||||
const maxW = cellW - 2 * LABEL_PAD;
|
||||
const maxH = cellH - LABEL_HEIGHT - 2 * LABEL_PAD;
|
||||
|
||||
const resized = await sharp(item.rawBuffer)
|
||||
.rotate()
|
||||
.resize(maxW, maxH, { fit: 'inside', withoutEnlargement: true })
|
||||
.toBuffer({ resolveWithObject: true });
|
||||
|
||||
overlays.push({
|
||||
input: resized.data,
|
||||
left: cellX + LABEL_PAD + Math.round((maxW - resized.info.width) / 2),
|
||||
top: cellY + LABEL_PAD + Math.round((maxH - resized.info.height) / 2),
|
||||
});
|
||||
|
||||
const labelW = cellW - 2 * LABEL_PAD;
|
||||
overlays.push({
|
||||
input: createLabelSvg(item.name, labelW, LABEL_HEIGHT, cols === 2 ? 12 : 11),
|
||||
left: cellX + LABEL_PAD,
|
||||
top: cellY + cellH - LABEL_HEIGHT - LABEL_PAD,
|
||||
});
|
||||
}
|
||||
|
||||
return sharp({
|
||||
create: {
|
||||
width: totalWidth,
|
||||
height: totalHeight,
|
||||
channels: 4,
|
||||
background: { r: 20, g: 21, b: 24, alpha: 1 },
|
||||
},
|
||||
})
|
||||
.composite(overlays)
|
||||
.webp({ quality: WEBP_QUALITY })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalises the caller's arguments into a list of `{ path, name }` items.
|
||||
*
|
||||
* Deliberately permissive on input shape (`path` / `paths` / `file` / `images`, string or
|
||||
* array, comma-separated) because models are inconsistent here, while keeping the schema
|
||||
* itself strict.
|
||||
*
|
||||
* @param {object} args
|
||||
* @returns {Array<{ path: string, name: string }>}
|
||||
*/
|
||||
function extractImageInputs(args = {}) {
|
||||
const items = [];
|
||||
|
||||
const add = (entry) => {
|
||||
if (!entry) return;
|
||||
if (typeof entry === 'string') {
|
||||
const trimmed = entry.trim();
|
||||
if (!trimmed) return;
|
||||
|
||||
// Some models send a JSON array as a string.
|
||||
if (trimmed.startsWith('[') && trimmed.endsWith(']')) {
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
if (Array.isArray(parsed)) {
|
||||
parsed.forEach(add);
|
||||
return;
|
||||
}
|
||||
} catch { /* not JSON, treat as a plain path */ }
|
||||
}
|
||||
|
||||
if (trimmed.includes(',')) {
|
||||
trimmed.split(',').map((s) => s.trim()).filter(Boolean).forEach((p) => {
|
||||
items.push({ path: p, name: path.posix.basename(p) || p });
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
items.push({ path: trimmed, name: path.posix.basename(trimmed) || trimmed });
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof entry === 'object') {
|
||||
const p = String(entry.path || entry.file || entry.filename || '').trim();
|
||||
if (!p) return;
|
||||
const name = String(entry.name || entry.title || path.posix.basename(p) || p).trim();
|
||||
items.push({ path: p, name });
|
||||
}
|
||||
};
|
||||
|
||||
for (const key of ['paths', 'images', 'path', 'file', 'filename']) {
|
||||
const value = args[key];
|
||||
if (Array.isArray(value)) value.forEach(add);
|
||||
else if (typeof value === 'string') add(value);
|
||||
}
|
||||
|
||||
if (items.length === 1 && typeof args.name === 'string' && args.name.trim()) {
|
||||
items[0].name = args.name.trim();
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches a database image by its content hash, authenticated as the calling user.
|
||||
* @param {string} hash 64-char SHA-256 hex
|
||||
* @param {number} googleAuthId
|
||||
* @returns {Promise<Buffer>}
|
||||
*/
|
||||
async function fetchDatabaseImage(hash, googleAuthId) {
|
||||
const url = `http://127.0.0.1:${env.port}/image/${hash}`;
|
||||
const token = jwt.sign({ id: googleAuthId }, env.jwtSecret, { expiresIn: '5m' });
|
||||
const res = await fetch(url, {
|
||||
headers: { Cookie: `token=${token}` },
|
||||
signal: AbortSignal.timeout(IMAGE_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
|
||||
return Buffer.from(await res.arrayBuffer());
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the analyze_image tool.
|
||||
* @param {{ googleAuthId: number, model: string }} ctx
|
||||
* `model` is the vision model id from the conversation record; the same model that is
|
||||
* driving the conversation is used for inspection so the agent does not silently switch
|
||||
* vendors mid-task.
|
||||
*/
|
||||
export function createAnalyzeImageTool({ googleAuthId, model }) {
|
||||
return defineTool({
|
||||
name: 'analyze_image',
|
||||
description:
|
||||
'Look at one or up to 9 images with a vision model and get its written judgement back. ' +
|
||||
'Paths are VFS-relative (e.g. "images/bild.de-1.avif" from search_images) or ' +
|
||||
'"image/<fullHash>" for a database image. ' +
|
||||
'Passing several paths renders them as a labelled grid in one call — use that to compare ' +
|
||||
'search candidates side by side. The returned text is the vision model\'s answer, so judge from it ' +
|
||||
'rather than claiming to have seen the image yourself.',
|
||||
strict: false,
|
||||
parameters: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['paths'],
|
||||
properties: {
|
||||
paths: {
|
||||
type: 'array',
|
||||
description:
|
||||
'1–9 image paths. VFS files (e.g. ["images/a-1.avif"]) or database images (e.g. ["image/<fullHash>"]). ' +
|
||||
'Several paths are rendered as one labelled grid.',
|
||||
items: { type: 'string' },
|
||||
minItems: 1,
|
||||
maxItems: MAX_IMAGES,
|
||||
},
|
||||
prompt: {
|
||||
type: 'string',
|
||||
description:
|
||||
'What to inspect or compare, e.g. "Which of these is a clean product shot on white, ' +
|
||||
'without watermark or text overlay?"',
|
||||
},
|
||||
grid: {
|
||||
type: 'string',
|
||||
enum: Object.keys(GRID_CONFIGS),
|
||||
description: 'Force a grid layout. Omit to pick one from the image count.',
|
||||
},
|
||||
},
|
||||
},
|
||||
invoke: async ({ paths, prompt, grid } = {}) => {
|
||||
const inputs = extractImageInputs({ paths });
|
||||
if (inputs.length === 0) {
|
||||
throw new ToolError(
|
||||
'analyze_image requires "paths": an array of 1–9 VFS paths (e.g. ["images/a-1.avif"]) ' +
|
||||
'or database paths (e.g. ["image/<fullHash>"]). Get candidates from search_images first.'
|
||||
);
|
||||
}
|
||||
if (inputs.length > MAX_IMAGES) {
|
||||
throw new ToolError(`analyze_image takes at most ${MAX_IMAGES} images per call (received ${inputs.length}). Split into batches.`);
|
||||
}
|
||||
|
||||
const vfsRoot = await prepareVfs({ googleAuthId });
|
||||
|
||||
// ── Load the bytes ────────────────────────────────────────────
|
||||
const items = [];
|
||||
for (const item of inputs) {
|
||||
// Only a path that actually carries a 64-char content hash is a database image.
|
||||
// A bare "images/…" prefix is just the VFS folder, so it must NOT be treated as
|
||||
// a DB lookup or `images/candidates/x.avif` would be misread as a hash path.
|
||||
const hash = extractFullHash(item.path);
|
||||
let buf = null;
|
||||
|
||||
if (!hash) {
|
||||
const abs = path.resolve(vfsRoot, item.path.replace(/^([a-zA-Z]:|\/)+/, '').replace(/^\.\/+/, ''));
|
||||
const rel = path.relative(vfsRoot, abs);
|
||||
if (rel.startsWith('..') || path.isAbsolute(rel)) {
|
||||
throw new ToolError(`Access denied: path "${item.path}" escapes virtual filesystem boundary.`);
|
||||
}
|
||||
if (fs.existsSync(abs) && fs.statSync(abs).isFile()) {
|
||||
buf = fs.readFileSync(abs);
|
||||
}
|
||||
} else {
|
||||
const cached = path.join(vfsRoot, 'images', `${hash}.avif`);
|
||||
if (fs.existsSync(cached)) {
|
||||
buf = fs.readFileSync(cached);
|
||||
} else {
|
||||
buf = await fetchDatabaseImage(hash, googleAuthId);
|
||||
}
|
||||
}
|
||||
|
||||
if (!buf || buf.length === 0) {
|
||||
throw new ToolError(
|
||||
`Image not found or empty: "${item.path}". ` +
|
||||
(hash
|
||||
? 'Database images must be addressed as "image/<fullHash>" with a 64-char hash.'
|
||||
: 'Store it first (search_images writes to "images/…") or check the path with find_files.')
|
||||
);
|
||||
}
|
||||
items.push({ ...item, rawBuffer: buf });
|
||||
}
|
||||
|
||||
// ── Compose ───────────────────────────────────────────────────
|
||||
const shape = resolveGrid(items.length, grid);
|
||||
let webp;
|
||||
if (items.length === 1 && shape.name === '1x1') {
|
||||
webp = await sharp(items[0].rawBuffer)
|
||||
.rotate()
|
||||
.resize(SINGLE_BOX, SINGLE_BOX, { fit: 'inside', withoutEnlargement: true })
|
||||
.webp({ quality: WEBP_QUALITY })
|
||||
.toBuffer();
|
||||
} else {
|
||||
webp = await renderGrid(items, GRID_CONFIGS[shape.name]);
|
||||
}
|
||||
|
||||
// ── Ask the vision model ──────────────────────────────────────
|
||||
const question = typeof prompt === 'string' && prompt.trim() ? prompt.trim() : null;
|
||||
const text = items.length > 1
|
||||
? `${question || 'Compare these images, judge visual quality, flag watermarks, text overlays or defects, and say which one is best and why.'}\n\n` +
|
||||
`The grid is ${shape.name}. Each image has its file name printed beneath it:\n` +
|
||||
items.map((it, idx) => {
|
||||
const row = Math.floor(idx / GRID_CONFIGS[shape.name].cols) + 1;
|
||||
const col = (idx % GRID_CONFIGS[shape.name].cols) + 1;
|
||||
return `- Row ${row}, Col ${col}: "${it.name}"`;
|
||||
}).join('\n')
|
||||
: question || 'Describe this image, judge its visual quality, flag any watermark, text overlay or defect, and say whether it is usable as a product or category image.';
|
||||
|
||||
const apiKey = await resolveOpenRouterKey(googleAuthId);
|
||||
const provider = new OpenRouterProvider({ apiKey });
|
||||
const response = await provider.chatCompletions({
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text },
|
||||
{ type: 'image_url', imageUrl: { url: `data:image/webp;base64,${webp.toString('base64')}` } },
|
||||
],
|
||||
},
|
||||
],
|
||||
signal: AbortSignal.timeout(120_000),
|
||||
});
|
||||
|
||||
const analysis = (response?.content || '').trim();
|
||||
return {
|
||||
images: items.map((it) => it.path),
|
||||
grid: shape.name,
|
||||
prompt: text,
|
||||
analysis: analysis || '(The vision model returned no text.)',
|
||||
usage: response?.usage
|
||||
? { input: response.usage.input, output: response.usage.output, cost: response.usage.cost }
|
||||
: undefined,
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts a 64-char hex SHA-256 hash from a VFS image path, if present.
|
||||
* Mirrors the sandbox's `extractFullHash` so `image/<hash>` resolves identically here.
|
||||
* @param {string} p
|
||||
* @returns {string|null}
|
||||
*/
|
||||
function extractFullHash(p) {
|
||||
if (typeof p !== 'string') return null;
|
||||
const parts = p.trim().replace(/^([a-zA-Z]:|\/)+/, '').split('/');
|
||||
for (const part of parts) {
|
||||
const seg = part.replace(/\.[a-zA-Z0-9]+$/, '');
|
||||
if (/^[a-fA-F0-9]{64}$/.test(seg)) return seg.toLowerCase();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
Reference in New Issue
Block a user