427 lines
17 KiB
JavaScript
427 lines
17 KiB
JavaScript
import fs from 'fs';
|
||
import path from 'path';
|
||
import sharp from 'sharp';
|
||
import jwt from 'jsonwebtoken';
|
||
import { defineTool, ToolError, OpenRouterProvider } from 'libllmorbiter';
|
||
import env from '../config/env.js';
|
||
import { getImageByHash } from '../db/pic.js';
|
||
import { prepareVfs, extractFullHash } from './vfs.js';
|
||
import { resolveOpenRouterKey, resolveDefaultModel } from './llm-key.js';
|
||
import {
|
||
TOOL_DESCRIPTIONS,
|
||
TOOL_PARAMETER_DESCRIPTIONS,
|
||
ANALYZE_IMAGE_VISION_COMPARE_PROMPT,
|
||
ANALYZE_IMAGE_VISION_DESCRIBE_PROMPT,
|
||
} from './prompts.js';
|
||
|
||
/**
|
||
* analyze_image — lets a vision model actually LOOK at images held in the VFS.
|
||
*
|
||
* The agent itself is text-only, so "seeing" an image requires a second, tool-free model
|
||
* call with the image attached. This tool is that bridge:
|
||
*
|
||
* - 1 image → resized to fit 400×400 and sent as a single WebP.
|
||
* - 2–9 → composited into a labelled grid (2x1/3x1/2x2/3x2/3x3) with the file name
|
||
* printed under each cell, so one cheap call can compare many candidates.
|
||
* - Database images are addressed as `image/<fullHash>` and fetched from `/image/:hash`.
|
||
*
|
||
* The prompt is assembled from the caller's `prompt` plus a layout summary, so the model
|
||
* knows which cell is which file.
|
||
*/
|
||
|
||
const GRID_CONFIGS = {
|
||
'1x1': { cols: 1, rows: 1, totalWidth: 400, totalHeight: 400 },
|
||
'2x1': { cols: 2, rows: 1, totalWidth: 600, totalHeight: 300 },
|
||
'3x1': { cols: 3, rows: 1, totalWidth: 800, totalHeight: 300 },
|
||
'2x2': { cols: 2, rows: 2, totalWidth: 600, totalHeight: 600 },
|
||
'3x2': { cols: 3, rows: 2, totalWidth: 800, totalHeight: 600 },
|
||
'3x3': { cols: 3, rows: 3, totalWidth: 800, totalHeight: 800 },
|
||
};
|
||
|
||
const MAX_IMAGES = 9;
|
||
const SINGLE_BOX = 400;
|
||
const WEBP_QUALITY = 80;
|
||
const IMAGE_TIMEOUT_MS = 10_000;
|
||
const LABEL_HEIGHT = 26;
|
||
const LABEL_PAD = 5;
|
||
|
||
/**
|
||
* Picks a grid shape: the caller's choice when valid, otherwise one that fits the count.
|
||
* @param {number} count
|
||
* @param {string} [requested]
|
||
* @returns {{ cols: number, rows: number, name: string }}
|
||
*/
|
||
function resolveGrid(count, requested) {
|
||
if (requested) {
|
||
const match = String(requested).trim().toLowerCase();
|
||
if (GRID_CONFIGS[match]) {
|
||
const [c, r] = match.split('x').map(Number);
|
||
return { cols: c, rows: r, name: match };
|
||
}
|
||
}
|
||
if (count <= 1) return { cols: 1, rows: 1, name: '1x1' };
|
||
if (count === 2) return { cols: 2, rows: 1, name: '2x1' };
|
||
if (count === 3) return { cols: 3, rows: 1, name: '3x1' };
|
||
if (count === 4) return { cols: 2, rows: 2, name: '2x2' };
|
||
if (count <= 6) return { cols: 3, rows: 2, name: '3x2' };
|
||
return { cols: 3, rows: 3, name: '3x3' };
|
||
}
|
||
|
||
/**
|
||
* Escapes text for safe embedding in the label SVG.
|
||
* @param {string} unsafe
|
||
* @returns {string}
|
||
*/
|
||
function escapeXml(unsafe) {
|
||
return String(unsafe).replace(/[<>&'"]/g, (c) => {
|
||
switch (c) {
|
||
case '<': return '<';
|
||
case '>': return '>';
|
||
case '&': return '&';
|
||
case "'": return ''';
|
||
case '"': return '"';
|
||
default: return c;
|
||
}
|
||
});
|
||
}
|
||
|
||
/**
|
||
* Shortens a label for display, keeping the extension visible so `a.avif` and `b.avif`
|
||
* never truncate to the same text.
|
||
* @param {string} name
|
||
* @param {number} maxChars
|
||
* @returns {string}
|
||
*/
|
||
function formatLabelText(name, maxChars = 28) {
|
||
if (!name || name.length <= maxChars) return name || '';
|
||
const extMatch = name.match(/(\.[a-zA-Z0-9]{3,5})$/);
|
||
if (extMatch) {
|
||
const ext = extMatch[1];
|
||
const avail = maxChars - ext.length - 1;
|
||
if (avail > 5) return `${name.slice(0, avail)}…${ext}`;
|
||
}
|
||
return `${name.slice(0, maxChars - 1)}…`;
|
||
}
|
||
|
||
/**
|
||
* Renders the dark caption bar drawn beneath each grid cell.
|
||
* @param {string} name
|
||
* @param {number} width
|
||
* @param {number} height
|
||
* @param {number} fontSize
|
||
* @returns {Buffer}
|
||
*/
|
||
function createLabelSvg(name, width, height, fontSize = 11) {
|
||
const maxChars = Math.max(8, Math.floor(width / (fontSize * 0.65)));
|
||
const text = escapeXml(formatLabelText(name, maxChars));
|
||
const textY = Math.round(height / 2 + fontSize / 3 + 1);
|
||
return Buffer.from(
|
||
`<svg width="${width}" height="${height}" xmlns="http://www.w3.org/2000/svg">` +
|
||
`<rect width="100%" height="100%" fill="#1e2025" rx="3" stroke="#333842" stroke-width="1"/>` +
|
||
`<text x="${Math.round(width / 2)}" y="${textY}" font-family="system-ui,-apple-system,sans-serif" ` +
|
||
`font-size="${fontSize}" font-weight="600" fill="#f1f5f9" text-anchor="middle">${text}</text>` +
|
||
`</svg>`
|
||
);
|
||
}
|
||
|
||
/**
|
||
* Composites the images into a single labelled grid WebP.
|
||
* @param {Array<{ name: string, rawBuffer: Buffer }>} items
|
||
* @param {{ cols: number, rows: number, totalWidth: number, totalHeight: number }} config
|
||
* @returns {Promise<Buffer>}
|
||
*/
|
||
async function renderGrid(items, config) {
|
||
const { cols, totalWidth, totalHeight } = config;
|
||
const overlays = [];
|
||
|
||
for (let i = 0; i < items.length; i += 1) {
|
||
const item = items[i];
|
||
const col = i % cols;
|
||
const row = Math.floor(i / cols);
|
||
|
||
const cellX = Math.round((col * totalWidth) / cols);
|
||
const cellW = Math.round(((col + 1) * totalWidth) / cols) - cellX;
|
||
const cellY = Math.round((row * totalHeight) / config.rows);
|
||
const cellH = Math.round(((row + 1) * totalHeight) / config.rows) - cellY;
|
||
|
||
const maxW = cellW - 2 * LABEL_PAD;
|
||
const maxH = cellH - LABEL_HEIGHT - 2 * LABEL_PAD;
|
||
|
||
const resized = await sharp(item.rawBuffer)
|
||
.rotate()
|
||
.resize(maxW, maxH, { fit: 'inside', withoutEnlargement: true })
|
||
.toBuffer({ resolveWithObject: true });
|
||
|
||
overlays.push({
|
||
input: resized.data,
|
||
left: cellX + LABEL_PAD + Math.round((maxW - resized.info.width) / 2),
|
||
top: cellY + LABEL_PAD + Math.round((maxH - resized.info.height) / 2),
|
||
});
|
||
|
||
const labelW = cellW - 2 * LABEL_PAD;
|
||
overlays.push({
|
||
input: createLabelSvg(item.name, labelW, LABEL_HEIGHT, cols === 2 ? 12 : 11),
|
||
left: cellX + LABEL_PAD,
|
||
top: cellY + cellH - LABEL_HEIGHT - LABEL_PAD,
|
||
});
|
||
}
|
||
|
||
return sharp({
|
||
create: {
|
||
width: totalWidth,
|
||
height: totalHeight,
|
||
channels: 4,
|
||
background: { r: 20, g: 21, b: 24, alpha: 1 },
|
||
},
|
||
})
|
||
.composite(overlays)
|
||
.webp({ quality: WEBP_QUALITY })
|
||
.toBuffer();
|
||
}
|
||
|
||
/**
|
||
* Normalises the caller's arguments into a list of `{ path, name }` items.
|
||
*
|
||
* Deliberately permissive on input shape (`path` / `paths` / `file` / `images`, string or
|
||
* array, comma-separated) because models are inconsistent here, while keeping the schema
|
||
* itself strict.
|
||
*
|
||
* @param {object} args
|
||
* @returns {Array<{ path: string, name: string }>}
|
||
*/
|
||
function extractImageInputs(args = {}) {
|
||
const items = [];
|
||
|
||
const add = (entry) => {
|
||
if (!entry) return;
|
||
if (typeof entry === 'string') {
|
||
const trimmed = entry.trim();
|
||
if (!trimmed) return;
|
||
|
||
// Some models send a JSON array as a string.
|
||
if (trimmed.startsWith('[') && trimmed.endsWith(']')) {
|
||
try {
|
||
const parsed = JSON.parse(trimmed);
|
||
if (Array.isArray(parsed)) {
|
||
parsed.forEach(add);
|
||
return;
|
||
}
|
||
} catch { /* not JSON, treat as a plain path */ }
|
||
}
|
||
|
||
if (trimmed.includes(',')) {
|
||
trimmed.split(',').map((s) => s.trim()).filter(Boolean).forEach((p) => {
|
||
items.push({ path: p, name: path.posix.basename(p) || p });
|
||
});
|
||
return;
|
||
}
|
||
|
||
items.push({ path: trimmed, name: path.posix.basename(trimmed) || trimmed });
|
||
return;
|
||
}
|
||
|
||
if (typeof entry === 'object') {
|
||
const p = String(entry.path || entry.file || entry.filename || '').trim();
|
||
if (!p) return;
|
||
const name = String(entry.name || entry.title || path.posix.basename(p) || p).trim();
|
||
items.push({ path: p, name });
|
||
}
|
||
};
|
||
|
||
for (const key of ['paths', 'images', 'path', 'file', 'filename']) {
|
||
const value = args[key];
|
||
if (Array.isArray(value)) value.forEach(add);
|
||
else if (typeof value === 'string') add(value);
|
||
}
|
||
|
||
if (items.length === 1 && typeof args.name === 'string' && args.name.trim()) {
|
||
items[0].name = args.name.trim();
|
||
}
|
||
return items;
|
||
}
|
||
|
||
/**
|
||
* Fetches a database image by its content hash using store or authenticated local endpoint.
|
||
* @param {string} hash 8-char SHA-256 Base62
|
||
* @param {number} googleAuthId
|
||
* @returns {Promise<Buffer>}
|
||
*/
|
||
async function fetchDatabaseImage(hash, googleAuthId) {
|
||
try {
|
||
const image = await getImageByHash(googleAuthId, hash);
|
||
if (image?.buffer) {
|
||
return image.buffer;
|
||
}
|
||
} catch (dbErr) {
|
||
try {
|
||
const url = `http://127.0.0.1:${env.port}/api/image/${hash}`;
|
||
const token = jwt.sign({ id: googleAuthId }, env.jwtSecret, { expiresIn: '5m' });
|
||
const res = await fetch(url, {
|
||
headers: { Cookie: `token=${token}` },
|
||
signal: AbortSignal.timeout(IMAGE_TIMEOUT_MS),
|
||
});
|
||
if (res.ok) {
|
||
return Buffer.from(await res.arrayBuffer());
|
||
}
|
||
} catch {
|
||
// fallback error ignored
|
||
}
|
||
throw dbErr;
|
||
}
|
||
throw new Error('Image not found');
|
||
}
|
||
|
||
/**
|
||
* Creates the analyze_image tool.
|
||
* @param {{ googleAuthId: number, model: string }} ctx
|
||
* `model` is the vision model id from the conversation record; the same model that is
|
||
* driving the conversation is used for inspection so the agent does not silently switch
|
||
* vendors mid-task.
|
||
*/
|
||
export function createAnalyzeImageTool({ googleAuthId, model }) {
|
||
return defineTool({
|
||
name: 'analyze_image',
|
||
description: TOOL_DESCRIPTIONS.analyze_image,
|
||
strict: false,
|
||
parameters: {
|
||
type: 'object',
|
||
additionalProperties: false,
|
||
required: ['paths'],
|
||
properties: {
|
||
note: {
|
||
type: ['string', 'null'],
|
||
description: TOOL_PARAMETER_DESCRIPTIONS.analyze_image.note,
|
||
},
|
||
paths: {
|
||
type: 'array',
|
||
description: TOOL_PARAMETER_DESCRIPTIONS.analyze_image.paths,
|
||
items: { type: 'string' },
|
||
minItems: 1,
|
||
maxItems: MAX_IMAGES,
|
||
},
|
||
prompt: {
|
||
type: ['string', 'null'],
|
||
description: TOOL_PARAMETER_DESCRIPTIONS.analyze_image.prompt,
|
||
},
|
||
grid: {
|
||
type: ['string', 'null'],
|
||
enum: [...Object.keys(GRID_CONFIGS), null],
|
||
description: TOOL_PARAMETER_DESCRIPTIONS.analyze_image.grid,
|
||
},
|
||
},
|
||
},
|
||
invoke: async ({ paths, prompt, grid, note } = {}) => {
|
||
const inputs = extractImageInputs({ paths });
|
||
if (inputs.length === 0) {
|
||
throw new ToolError(
|
||
'analyze_image requires "paths": an array of 1–9 VFS paths (e.g. ["images/a-1.avif"]) ' +
|
||
'or database paths (e.g. ["image/<fullHash>"]). Get candidates from search_images first.'
|
||
);
|
||
}
|
||
if (inputs.length > MAX_IMAGES) {
|
||
throw new ToolError(`analyze_image takes at most ${MAX_IMAGES} images per call (received ${inputs.length}). Split into batches.`);
|
||
}
|
||
|
||
const vfsRoot = await prepareVfs({ googleAuthId });
|
||
|
||
// ── Load the bytes ────────────────────────────────────────────
|
||
const items = [];
|
||
for (const item of inputs) {
|
||
// Only a path that actually carries an 8-char content hash is a database image.
|
||
// A bare "images/…" prefix is just the VFS folder, so it must NOT be treated as
|
||
// a DB lookup or `images/candidates/x.avif` would be misread as a hash path.
|
||
const hash = extractFullHash(item.path);
|
||
let buf = null;
|
||
|
||
if (!hash) {
|
||
const abs = path.resolve(vfsRoot, item.path.replace(/^([a-zA-Z]:|\/)+/, '').replace(/^\.\/+/, ''));
|
||
const rel = path.relative(vfsRoot, abs);
|
||
if (rel.startsWith('..') || path.isAbsolute(rel)) {
|
||
throw new ToolError(`Access denied: path "${item.path}" escapes virtual filesystem boundary.`);
|
||
}
|
||
if (fs.existsSync(abs) && fs.statSync(abs).isFile()) {
|
||
buf = fs.readFileSync(abs);
|
||
}
|
||
} else {
|
||
const cached = path.join(vfsRoot, 'images', `${hash}.avif`);
|
||
if (fs.existsSync(cached)) {
|
||
buf = fs.readFileSync(cached);
|
||
} else {
|
||
buf = await fetchDatabaseImage(hash, googleAuthId);
|
||
try {
|
||
fs.mkdirSync(path.join(vfsRoot, 'images'), { recursive: true });
|
||
fs.writeFileSync(cached, buf);
|
||
} catch { /* ignore cache write failure */ }
|
||
}
|
||
}
|
||
|
||
if (!buf || buf.length === 0) {
|
||
throw new ToolError(
|
||
`Image not found or empty: "${item.path}". ` +
|
||
(hash
|
||
? 'Database images must be addressed as "image/<fullHash>" with an 8-char hash.'
|
||
: 'Store it first (search_images writes to "images/…") or check the path with find_files.')
|
||
);
|
||
}
|
||
items.push({ ...item, rawBuffer: buf });
|
||
}
|
||
|
||
// ── Compose ───────────────────────────────────────────────────
|
||
const shape = resolveGrid(items.length, grid);
|
||
let webp;
|
||
if (items.length === 1 && shape.name === '1x1') {
|
||
webp = await sharp(items[0].rawBuffer)
|
||
.rotate()
|
||
.resize(SINGLE_BOX, SINGLE_BOX, { fit: 'inside', withoutEnlargement: true })
|
||
.webp({ quality: WEBP_QUALITY })
|
||
.toBuffer();
|
||
} else {
|
||
webp = await renderGrid(items, GRID_CONFIGS[shape.name]);
|
||
}
|
||
|
||
// ── Ask the vision model ──────────────────────────────────────
|
||
const question = typeof prompt === 'string' && prompt.trim() ? prompt.trim() : null;
|
||
const text = items.length > 1
|
||
? `${question || ANALYZE_IMAGE_VISION_COMPARE_PROMPT}\n\n` +
|
||
`The grid is ${shape.name}. Each image has its file name printed beneath it:\n` +
|
||
items.map((it, idx) => {
|
||
const row = Math.floor(idx / GRID_CONFIGS[shape.name].cols) + 1;
|
||
const col = (idx % GRID_CONFIGS[shape.name].cols) + 1;
|
||
return `- Row ${row}, Col ${col}: "${it.name}"`;
|
||
}).join('\n')
|
||
: question || ANALYZE_IMAGE_VISION_DESCRIBE_PROMPT;
|
||
|
||
const apiKey = await resolveOpenRouterKey(googleAuthId);
|
||
const visionModel = (await resolveDefaultModel(googleAuthId)) || model;
|
||
const provider = new OpenRouterProvider({ apiKey, apiUrl: env.openrouterApiUrl });
|
||
const response = await provider.chatCompletions({
|
||
model: visionModel,
|
||
messages: [
|
||
{
|
||
role: 'user',
|
||
content: [
|
||
{ type: 'text', text },
|
||
{
|
||
type: 'image_url',
|
||
image_url: { url: `data:image/webp;base64,${webp.toString('base64')}` },
|
||
},
|
||
],
|
||
},
|
||
],
|
||
signal: AbortSignal.timeout(120_000),
|
||
});
|
||
|
||
const analysis = (response?.content || '').trim();
|
||
return {
|
||
images: items.map((it) => it.path),
|
||
grid: shape.name,
|
||
prompt: text,
|
||
analysis: analysis || '(The vision model returned no text.)',
|
||
usage: response?.usage
|
||
? { input: response.usage.input, output: response.usage.output, cost: response.usage.cost }
|
||
: undefined,
|
||
};
|
||
},
|
||
});
|
||
}
|