import fs from 'fs'; import path from 'path'; import { optimizeImage } from 'wasm-image-optimization'; import { defineTool, ToolError } from 'libllmorbiter'; import * as dbCompanySetting from '../db/companysetting.js'; import { prepareVfs, syncVfsToDb } from './vfs.js'; import { TOOL_DESCRIPTIONS, TOOL_PARAMETER_DESCRIPTIONS } from './prompts.js'; const SERPER_ENDPOINT = 'https://google.serper.dev/images'; const SEARCH_TIMEOUT_MS = 15_000; const DOWNLOAD_TIMEOUT_MS = 8_000; const MAX_EDGE = 1000; const AVIF_QUALITY = 80; const DEFAULT_COUNT = 4; const MAX_COUNT = 9; const IMAGE_FOLDER = 'images'; /** * search_images — Google image search via Serper.dev, normalised into the VFS. * * Results are written to the VFS as AVIF candidates and returned as paths, NOT as inline * base64 blobs. That is the whole point: the model can hand several candidates to * `analyze_image`, which grids them into one vision call, and only the chosen one is * committed to the database. Dumping 9 images into the transcript would be both enormous * and useless, because the model cannot look at an image without asking the vision model to. * * Every downloaded candidate lands in `images/-.avif` and is persisted to * `modelvfs`, so a follow-up turn can still reference them. */ const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'; const ACCEPT_HEADER = 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8'; /** * Slugifies a hostname into a safe, readable file-name fragment. * @param {string} host * @returns {string} */ function slugifyHost(host) { const cleaned = String(host || '') .toLowerCase() .replace(/^www\./, '') .replace(/[^a-z0-9.]+/g, '-') .replace(/-+/g, '-') .replace(/^-|-$/g, ''); return cleaned.slice(0, 40) || 'source'; } /** * Queries Serper for image results. * * @param {string} query * @param {string} apiKey * @param {{ gl?: string, hl?: string, signal?: AbortSignal }} [options] * @returns {Promise>} raw `images` array from Serper */ async function serperImageSearch(query, apiKey, { gl = 'de', hl = 'de', signal } = {}) { const headers = { 'X-API-KEY': apiKey, 'Content-Type': 'application/json' }; // Cap the whole search at SEARCH_TIMEOUT_MS even if the caller's signal never fires, so a // hung Serper connection cannot wedge the agent turn indefinitely. const timeoutSignal = AbortSignal.timeout(SEARCH_TIMEOUT_MS); const combined = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal; let response; try { response = await fetch(SERPER_ENDPOINT, { method: 'POST', headers, body: JSON.stringify({ q: query, gl, hl }), redirect: 'follow', signal: combined, }); } catch (err) { throw new ToolError(`search_images: network failure contacting Serper (${query}): ${err.message}`); } if (!response.ok) { let detail = `${response.status} ${response.statusText}`; try { const body = await response.text(); if (body) detail += ` - ${body.slice(0, 300)}`; } catch { /* body already consumed or unreadable */ } throw new ToolError(`search_images: Serper API error: ${detail}`); } const data = await response.json(); return Array.isArray(data?.images) ? data.images : []; } /** * Downloads one candidate and converts it to a ratio-preserving AVIF buffer. * * @param {object} img raw Serper image entry * @param {{ signal?: AbortSignal }} [options] * @returns {Promise} AVIF bytes, or null if the image was unusable */ async function toAvif(img, { signal } = {}) { const urls = [img?.imageUrl, img?.thumbnailUrl].filter((u) => typeof u === 'string' && u.trim()); if (urls.length === 0) return null; for (const url of urls) { if (signal?.aborted) return null; try { const downloadSignal = AbortSignal.any( signal ? [signal, AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)] : [AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)] ); const res = await fetch(url, { headers: { 'User-Agent': USER_AGENT, Accept: ACCEPT_HEADER }, signal: downloadSignal, redirect: 'follow', }); if (!res.ok) continue; const input = Buffer.from(await res.arrayBuffer()); if (input.length === 0) continue; const meta = await optimizeImage({ image: input, format: 'none' }); const srcW = meta.originalWidth || 0; const srcH = meta.originalHeight || 0; const needsResize = srcW > MAX_EDGE || srcH > MAX_EDGE; const scale = needsResize ? Math.min(MAX_EDGE / srcW, MAX_EDGE / srcH) : 1; const out = await optimizeImage({ image: input, width: needsResize ? Math.max(1, Math.round(srcW * scale)) : undefined, height: needsResize ? Math.max(1, Math.round(srcH * scale)) : undefined, fit: 'fill', format: 'avif', quality: AVIF_QUALITY, }); return Buffer.from(out.data); } catch { // Try the next candidate URL; a dead imageUrl with a live thumbnail is common. } } return null; } /** * Creates the search_images tool bound to a company's virtual filesystem. * @param {{ googleAuthId: number }} ctx */ export function createSearchImagesTool({ googleAuthId }) { return defineTool({ name: 'search_images', description: TOOL_DESCRIPTIONS.search_images, strict: false, parameters: { type: 'object', additionalProperties: false, required: ['query'], properties: { note: { type: ['string', 'null'], description: TOOL_PARAMETER_DESCRIPTIONS.search_images.note, }, query: { type: 'string', description: TOOL_PARAMETER_DESCRIPTIONS.search_images.query, }, count: { type: ['number', 'null'], description: TOOL_PARAMETER_DESCRIPTIONS.search_images.count, }, gl: { type: ['string', 'null'], description: TOOL_PARAMETER_DESCRIPTIONS.search_images.gl, }, hl: { type: ['string', 'null'], description: TOOL_PARAMETER_DESCRIPTIONS.search_images.hl, }, }, }, invoke: async ({ query, count = DEFAULT_COUNT, gl = 'de', hl = 'de', note } = {}) => { const term = typeof query === 'string' ? query.trim() : ''; if (!term) { throw new ToolError('search_images requires a non-empty "query" (e.g. " Mineralwasser Flasche Glas ").'); } const resolvedCount = count != null ? Number(count) : DEFAULT_COUNT; if (!Number.isInteger(resolvedCount) || resolvedCount < 1) { throw new ToolError(`"count" must be a positive integer (received ${count}).`); } const wanted = Math.min(resolvedCount, MAX_COUNT); const resolvedGl = gl || 'de'; const resolvedHl = hl || 'de'; let apiKey = null; try { const setting = await dbCompanySetting.getByKey(googleAuthId, 'keys.serper', { maskWriteonly: false }); apiKey = setting?.value?.trim() || null; } catch (err) { console.warn('search_images: failed to read company setting keys.serper:', err?.message || err); } if (!apiKey) { throw new ToolError( 'search_images is unavailable: Serper API key not configured (companysetting: keys.serper). ' + 'Use a direct image URL with javascript_execute instead.' ); } const raw = await serperImageSearch(term, apiKey, { gl: resolvedGl, hl: resolvedHl }); if (raw.length === 0) { return { query: term, total: 0, returned: 0, images: [], note: `No image results for "${term}". Try a broader or more generic term.`, }; } const vfsRoot = await prepareVfs({ googleAuthId }); const imagesDir = path.join(vfsRoot, IMAGE_FOLDER); fs.mkdirSync(imagesDir, { recursive: true }); const results = []; const failures = []; // Take a generous slice from Serper: downloads fail often enough (hotlink // protection, dead URLs) that stopping at `wanted` raw hits under-fills the batch. for (const img of raw.slice(0, Math.max(wanted * 3, wanted))) { if (results.length >= wanted) break; const host = img?.domain || (() => { try { return img?.imageUrl ? new URL(img.imageUrl).hostname : ''; } catch { return ''; } })(); const bytes = await toAvif(img); if (!bytes || bytes.length === 0) { failures.push({ source: img?.link || img?.imageUrl || '', reason: 'download or conversion failed' }); continue; } const name = `${slugifyHost(host)}-${results.length + 1}.avif`; const relPath = `${IMAGE_FOLDER}/${name}`; fs.writeFileSync(path.join(imagesDir, name), bytes); results.push({ path: relPath, domain: host, title: typeof img?.title === 'string' ? img.title : '', sourcePage: typeof img?.link === 'string' ? img.link : '', bytes: bytes.length, }); } if (results.length > 0) { try { await syncVfsToDb({ googleAuthId, vfsRoot }); } catch (err) { // Non-fatal: the files are on disk and will be picked up by the next // javascript_execute / patch sync. Losing the images because a DB write // failed would be a worse outcome than a slightly stale store. console.warn('[search_images] failed to persist candidates:', err.message); } } const result = { query: term, total: results.length, returned: results.length, images: results, nextStep: 'Call analyze_image with these paths (grid) to compare them, then upload the winner.', }; if (failures.length > 0) result.skipped = failures.length; if (results.length === 0) { result.note = `Found ${raw.length} result(s) for "${term}" but none could be downloaded. Try different wording.`; } return result; }, }); }