u

src/tools/search-images.js

@@ -0,0 +1,257 @@
+import fs from 'fs';
+import path from 'path';
+import sharp from 'sharp';
+import { defineTool, ToolError } from 'libllmorbiter';
+import env from '../config/env.js';
+import { prepareVfs, syncVfsToDb } from './vfs.js';
+
+const SERPER_ENDPOINT = 'https://google.serper.dev/images';
+const SEARCH_TIMEOUT_MS = 15_000;
+const DOWNLOAD_TIMEOUT_MS = 8_000;
+const MAX_EDGE = 1000;
+const AVIF_QUALITY = 80;
+const DEFAULT_COUNT = 4;
+const MAX_COUNT = 9;
+
+const IMAGE_FOLDER = 'images';
+
+/**
+ * search_images — Google image search via Serper.dev, normalised into the VFS.
+ *
+ * Results are written to the VFS as AVIF candidates and returned as paths, NOT as inline
+ * base64 blobs. That is the whole point: the model can hand several candidates to
+ * `analyze_image`, which grids them into one vision call, and only the chosen one is
+ * committed to the database. Dumping 9 images into the transcript would be both enormous
+ * and useless, because the model cannot look at an image without asking the vision model to.
+ *
+ * Every downloaded candidate lands in `images/<domain>-<n>.avif` and is persisted to
+ * `modelvfs`, so a follow-up turn can still reference them.
+ */
+
+const USER_AGENT =
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+
+const ACCEPT_HEADER = 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8';
+
+/**
+ * Slugifies a hostname into a safe, readable file-name fragment.
+ * @param {string} host
+ * @returns {string}
+ */
+function slugifyHost(host) {
+    const cleaned = String(host || '')
+        .toLowerCase()
+        .replace(/^www\./, '')
+        .replace(/[^a-z0-9.]+/g, '-')
+        .replace(/-+/g, '-')
+        .replace(/^-|-$/g, '');
+    return cleaned.slice(0, 40) || 'source';
+}
+
+/**
+ * Queries Serper for image results.
+ *
+ * @param {string} query
+ * @param {string} apiKey
+ * @param {{ gl?: string, hl?: string, signal?: AbortSignal }} [options]
+ * @returns {Promise<Array<object>>} raw `images` array from Serper
+ */
+async function serperImageSearch(query, apiKey, { gl = 'de', hl = 'de', signal } = {}) {
+    const headers = { 'X-API-KEY': apiKey, 'Content-Type': 'application/json' };
+
+    // Cap the whole search at SEARCH_TIMEOUT_MS even if the caller's signal never fires, so a
+    // hung Serper connection cannot wedge the agent turn indefinitely.
+    const timeoutSignal = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
+    const combined = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
+
+    let response;
+    try {
+        response = await fetch(SERPER_ENDPOINT, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({ q: query, gl, hl }),
+            redirect: 'follow',
+            signal: combined,
+        });
+    } catch (err) {
+        throw new ToolError(`search_images: network failure contacting Serper (${query}): ${err.message}`);
+    }
+
+    if (!response.ok) {
+        let detail = `${response.status} ${response.statusText}`;
+        try {
+            const body = await response.text();
+            if (body) detail += ` - ${body.slice(0, 300)}`;
+        } catch { /* body already consumed or unreadable */ }
+        throw new ToolError(`search_images: Serper API error: ${detail}`);
+    }
+
+    const data = await response.json();
+    return Array.isArray(data?.images) ? data.images : [];
+}
+
+/**
+ * Downloads one candidate and converts it to a ratio-preserving AVIF buffer.
+ *
+ * @param {object} img raw Serper image entry
+ * @param {{ signal?: AbortSignal }} [options]
+ * @returns {Promise<Buffer|null>} AVIF bytes, or null if the image was unusable
+ */
+async function toAvif(img, { signal } = {}) {
+    const urls = [img?.imageUrl, img?.thumbnailUrl].filter((u) => typeof u === 'string' && u.trim());
+    if (urls.length === 0) return null;
+
+    for (const url of urls) {
+        if (signal?.aborted) return null;
+        try {
+            const downloadSignal = AbortSignal.any(
+                signal ? [signal, AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)] : [AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)]
+            );
+            const res = await fetch(url, {
+                headers: { 'User-Agent': USER_AGENT, Accept: ACCEPT_HEADER },
+                signal: downloadSignal,
+                redirect: 'follow',
+            });
+            if (!res.ok) continue;
+
+            const input = Buffer.from(await res.arrayBuffer());
+            if (input.length === 0) continue;
+
+            return await sharp(input)
+                .rotate() // honour EXIF orientation before resizing
+                .resize(MAX_EDGE, MAX_EDGE, { fit: 'inside', withoutEnlargement: true })
+                .avif({ quality: AVIF_QUALITY })
+                .toBuffer();
+        } catch {
+            // Try the next candidate URL; a dead imageUrl with a live thumbnail is common.
+        }
+    }
+    return null;
+}
+
+/**
+ * Creates the search_images tool bound to a company's virtual filesystem.
+ * @param {{ googleAuthId: number }} ctx
+ */
+export function createSearchImagesTool({ googleAuthId }) {
+    return defineTool({
+        name: 'search_images',
+        description:
+            'Search Google Images and store the results in the virtual filesystem as AVIF candidates. ' +
+            'Returns VFS paths, not inline images — then pass those paths to analyze_image to actually look at them ' +
+            '(up to 9 at once, rendered as a labelled grid) and pick the best one before uploading it to an article ' +
+            'or category. Candidates are cached, so a second search of the same term is cheap.',
+        strict: false,
+        parameters: {
+            type: 'object',
+            additionalProperties: false,
+            required: ['query'],
+            properties: {
+                query: {
+                    type: 'string',
+                    description: 'Search term, e.g. " Mineralwasser Flasche Glas ".',
+                },
+                count: {
+                    type: 'number',
+                    description: `How many candidate images to return. Default ${DEFAULT_COUNT}, max ${MAX_COUNT}.`,
+                },
+                gl: {
+                    type: 'string',
+                    description: 'Geolocation country code. Default "de".',
+                },
+                hl: {
+                    type: 'string',
+                    description: 'Result language code. Default "de".',
+                },
+            },
+        },
+        invoke: async ({ query, count = DEFAULT_COUNT, gl = 'de', hl = 'de' } = {}) => {
+            const term = typeof query === 'string' ? query.trim() : '';
+            if (!term) {
+                throw new ToolError('search_images requires a non-empty "query" (e.g. " Mineralwasser Flasche Glas ").');
+            }
+            if (!Number.isInteger(count) || count < 1) {
+                throw new ToolError(`"count" must be a positive integer (received ${count}).`);
+            }
+            const wanted = Math.min(count, MAX_COUNT);
+
+            const apiKey = env.serperApiKey?.trim();
+            if (!apiKey) {
+                throw new ToolError(
+                    'search_images is unavailable: SERPER_API_KEY is not configured on the server. ' +
+                    'Use a direct image URL with javascript_execute instead.'
+                );
+            }
+
+            const raw = await serperImageSearch(term, apiKey, { gl, hl });
+            if (raw.length === 0) {
+                return {
+                    query: term,
+                    total: 0,
+                    returned: 0,
+                    images: [],
+                    note: `No image results for "${term}". Try a broader or more generic term.`,
+                };
+            }
+
+            const vfsRoot = await prepareVfs({ googleAuthId });
+            const imagesDir = path.join(vfsRoot, IMAGE_FOLDER);
+            fs.mkdirSync(imagesDir, { recursive: true });
+
+            const results = [];
+            const failures = [];
+
+            // Take a generous slice from Serper: downloads fail often enough (hotlink
+            // protection, dead URLs) that stopping at `wanted` raw hits under-fills the batch.
+            for (const img of raw.slice(0, Math.max(wanted * 3, wanted))) {
+                if (results.length >= wanted) break;
+
+                const host = img?.domain || (() => {
+                    try { return img?.imageUrl ? new URL(img.imageUrl).hostname : ''; } catch { return ''; }
+                })();
+
+                const bytes = await toAvif(img);
+                if (!bytes || bytes.length === 0) {
+                    failures.push({ source: img?.link || img?.imageUrl || '', reason: 'download or conversion failed' });
+                    continue;
+                }
+
+                const name = `${slugifyHost(host)}-${results.length + 1}.avif`;
+                const relPath = `${IMAGE_FOLDER}/${name}`;
+                fs.writeFileSync(path.join(imagesDir, name), bytes);
+
+                results.push({
+                    path: relPath,
+                    domain: host,
+                    title: typeof img?.title === 'string' ? img.title : '',
+                    sourcePage: typeof img?.link === 'string' ? img.link : '',
+                    bytes: bytes.length,
+                });
+            }
+
+            if (results.length > 0) {
+                try {
+                    await syncVfsToDb({ googleAuthId, vfsRoot });
+                } catch (err) {
+                    // Non-fatal: the files are on disk and will be picked up by the next
+                    // javascript_execute / patch sync. Losing the images because a DB write
+                    // failed would be a worse outcome than a slightly stale store.
+                    console.warn('[search_images] failed to persist candidates:', err.message);
+                }
+            }
+
+            const result = {
+                query: term,
+                total: results.length,
+                returned: results.length,
+                images: results,
+                nextStep: 'Call analyze_image with these paths (grid) to compare them, then upload the winner.',
+            };
+            if (failures.length > 0) result.skipped = failures.length;
+            if (results.length === 0) {
+                result.note = `Found ${raw.length} result(s) for "${term}" but none could be downloaded. Try different wording.`;
+            }
+            return result;
+        },
+    });
+}