277 lines
11 KiB
JavaScript
277 lines
11 KiB
JavaScript
import fs from 'fs';
|
|
import path from 'path';
|
|
import { optimizeImage } from 'wasm-image-optimization';
|
|
import { defineTool, ToolError } from 'libllmorbiter';
|
|
import * as dbCompanySetting from '../db/companysetting.js';
|
|
import { prepareVfs, syncVfsToDb } from './vfs.js';
|
|
import { TOOL_DESCRIPTIONS, TOOL_PARAMETER_DESCRIPTIONS } from './prompts.js';
|
|
|
|
const SERPER_ENDPOINT = 'https://google.serper.dev/images';
|
|
const SEARCH_TIMEOUT_MS = 15_000;
|
|
const DOWNLOAD_TIMEOUT_MS = 8_000;
|
|
const MAX_EDGE = 1000;
|
|
const AVIF_QUALITY = 80;
|
|
const DEFAULT_COUNT = 4;
|
|
const MAX_COUNT = 9;
|
|
|
|
const IMAGE_FOLDER = 'images';
|
|
|
|
/**
|
|
* search_images — Google image search via Serper.dev, normalised into the VFS.
|
|
*
|
|
* Results are written to the VFS as AVIF candidates and returned as paths, NOT as inline
|
|
* base64 blobs. That is the whole point: the model can hand several candidates to
|
|
* `analyze_image`, which grids them into one vision call, and only the chosen one is
|
|
* committed to the database. Dumping 9 images into the transcript would be both enormous
|
|
* and useless, because the model cannot look at an image without asking the vision model to.
|
|
*
|
|
* Every downloaded candidate lands in `images/<domain>-<n>.avif` and is persisted to
|
|
* `modelvfs`, so a follow-up turn can still reference them.
|
|
*/
|
|
|
|
const USER_AGENT =
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
|
|
|
const ACCEPT_HEADER = 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8';
|
|
|
|
/**
|
|
* Slugifies a hostname into a safe, readable file-name fragment.
|
|
* @param {string} host
|
|
* @returns {string}
|
|
*/
|
|
function slugifyHost(host) {
|
|
const cleaned = String(host || '')
|
|
.toLowerCase()
|
|
.replace(/^www\./, '')
|
|
.replace(/[^a-z0-9.]+/g, '-')
|
|
.replace(/-+/g, '-')
|
|
.replace(/^-|-$/g, '');
|
|
return cleaned.slice(0, 40) || 'source';
|
|
}
|
|
|
|
/**
|
|
* Queries Serper for image results.
|
|
*
|
|
* @param {string} query
|
|
* @param {string} apiKey
|
|
* @param {{ gl?: string, hl?: string, signal?: AbortSignal }} [options]
|
|
* @returns {Promise<Array<object>>} raw `images` array from Serper
|
|
*/
|
|
async function serperImageSearch(query, apiKey, { gl = 'de', hl = 'de', signal } = {}) {
|
|
const headers = { 'X-API-KEY': apiKey, 'Content-Type': 'application/json' };
|
|
|
|
// Cap the whole search at SEARCH_TIMEOUT_MS even if the caller's signal never fires, so a
|
|
// hung Serper connection cannot wedge the agent turn indefinitely.
|
|
const timeoutSignal = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
|
|
const combined = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
|
|
|
let response;
|
|
try {
|
|
response = await fetch(SERPER_ENDPOINT, {
|
|
method: 'POST',
|
|
headers,
|
|
body: JSON.stringify({ q: query, gl, hl }),
|
|
redirect: 'follow',
|
|
signal: combined,
|
|
});
|
|
} catch (err) {
|
|
throw new ToolError(`search_images: network failure contacting Serper (${query}): ${err.message}`);
|
|
}
|
|
|
|
if (!response.ok) {
|
|
let detail = `${response.status} ${response.statusText}`;
|
|
try {
|
|
const body = await response.text();
|
|
if (body) detail += ` - ${body.slice(0, 300)}`;
|
|
} catch { /* body already consumed or unreadable */ }
|
|
throw new ToolError(`search_images: Serper API error: ${detail}`);
|
|
}
|
|
|
|
const data = await response.json();
|
|
return Array.isArray(data?.images) ? data.images : [];
|
|
}
|
|
|
|
/**
|
|
* Downloads one candidate and converts it to a ratio-preserving AVIF buffer.
|
|
*
|
|
* @param {object} img raw Serper image entry
|
|
* @param {{ signal?: AbortSignal }} [options]
|
|
* @returns {Promise<Buffer|null>} AVIF bytes, or null if the image was unusable
|
|
*/
|
|
async function toAvif(img, { signal } = {}) {
|
|
const urls = [img?.imageUrl, img?.thumbnailUrl].filter((u) => typeof u === 'string' && u.trim());
|
|
if (urls.length === 0) return null;
|
|
|
|
for (const url of urls) {
|
|
if (signal?.aborted) return null;
|
|
try {
|
|
const downloadSignal = AbortSignal.any(
|
|
signal ? [signal, AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)] : [AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)]
|
|
);
|
|
const res = await fetch(url, {
|
|
headers: { 'User-Agent': USER_AGENT, Accept: ACCEPT_HEADER },
|
|
signal: downloadSignal,
|
|
redirect: 'follow',
|
|
});
|
|
if (!res.ok) continue;
|
|
|
|
const input = Buffer.from(await res.arrayBuffer());
|
|
if (input.length === 0) continue;
|
|
|
|
const meta = await optimizeImage({ image: input, format: 'none' });
|
|
const srcW = meta.originalWidth || 0;
|
|
const srcH = meta.originalHeight || 0;
|
|
const needsResize = srcW > MAX_EDGE || srcH > MAX_EDGE;
|
|
const scale = needsResize ? Math.min(MAX_EDGE / srcW, MAX_EDGE / srcH) : 1;
|
|
const out = await optimizeImage({
|
|
image: input,
|
|
width: needsResize ? Math.max(1, Math.round(srcW * scale)) : undefined,
|
|
height: needsResize ? Math.max(1, Math.round(srcH * scale)) : undefined,
|
|
fit: 'fill',
|
|
format: 'avif',
|
|
quality: AVIF_QUALITY,
|
|
});
|
|
return Buffer.from(out.data);
|
|
} catch {
|
|
// Try the next candidate URL; a dead imageUrl with a live thumbnail is common.
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Creates the search_images tool bound to a company's virtual filesystem.
|
|
* @param {{ googleAuthId: number }} ctx
|
|
*/
|
|
export function createSearchImagesTool({ googleAuthId }) {
|
|
return defineTool({
|
|
name: 'search_images',
|
|
description: TOOL_DESCRIPTIONS.search_images,
|
|
strict: false,
|
|
parameters: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
required: ['query'],
|
|
properties: {
|
|
note: {
|
|
type: ['string', 'null'],
|
|
description: TOOL_PARAMETER_DESCRIPTIONS.search_images.note,
|
|
},
|
|
query: {
|
|
type: 'string',
|
|
description: TOOL_PARAMETER_DESCRIPTIONS.search_images.query,
|
|
},
|
|
count: {
|
|
type: ['number', 'null'],
|
|
description: TOOL_PARAMETER_DESCRIPTIONS.search_images.count,
|
|
},
|
|
gl: {
|
|
type: ['string', 'null'],
|
|
description: TOOL_PARAMETER_DESCRIPTIONS.search_images.gl,
|
|
},
|
|
hl: {
|
|
type: ['string', 'null'],
|
|
description: TOOL_PARAMETER_DESCRIPTIONS.search_images.hl,
|
|
},
|
|
},
|
|
},
|
|
invoke: async ({ query, count = DEFAULT_COUNT, gl = 'de', hl = 'de', note } = {}) => {
|
|
const term = typeof query === 'string' ? query.trim() : '';
|
|
if (!term) {
|
|
throw new ToolError('search_images requires a non-empty "query" (e.g. " Mineralwasser Flasche Glas ").');
|
|
}
|
|
const resolvedCount = count != null ? Number(count) : DEFAULT_COUNT;
|
|
if (!Number.isInteger(resolvedCount) || resolvedCount < 1) {
|
|
throw new ToolError(`"count" must be a positive integer (received ${count}).`);
|
|
}
|
|
const wanted = Math.min(resolvedCount, MAX_COUNT);
|
|
const resolvedGl = gl || 'de';
|
|
const resolvedHl = hl || 'de';
|
|
|
|
let apiKey = null;
|
|
try {
|
|
const setting = await dbCompanySetting.getByKey(googleAuthId, 'keys.serper', { maskWriteonly: false });
|
|
apiKey = setting?.value?.trim() || null;
|
|
} catch (err) {
|
|
console.warn('search_images: failed to read company setting keys.serper:', err?.message || err);
|
|
}
|
|
|
|
if (!apiKey) {
|
|
throw new ToolError(
|
|
'search_images is unavailable: Serper API key not configured (companysetting: keys.serper). ' +
|
|
'Use a direct image URL with javascript_execute instead.'
|
|
);
|
|
}
|
|
|
|
const raw = await serperImageSearch(term, apiKey, { gl: resolvedGl, hl: resolvedHl });
|
|
if (raw.length === 0) {
|
|
return {
|
|
query: term,
|
|
total: 0,
|
|
returned: 0,
|
|
images: [],
|
|
note: `No image results for "${term}". Try a broader or more generic term.`,
|
|
};
|
|
}
|
|
|
|
const vfsRoot = await prepareVfs({ googleAuthId });
|
|
const imagesDir = path.join(vfsRoot, IMAGE_FOLDER);
|
|
fs.mkdirSync(imagesDir, { recursive: true });
|
|
|
|
const results = [];
|
|
const failures = [];
|
|
|
|
// Take a generous slice from Serper: downloads fail often enough (hotlink
|
|
// protection, dead URLs) that stopping at `wanted` raw hits under-fills the batch.
|
|
for (const img of raw.slice(0, Math.max(wanted * 3, wanted))) {
|
|
if (results.length >= wanted) break;
|
|
|
|
const host = img?.domain || (() => {
|
|
try { return img?.imageUrl ? new URL(img.imageUrl).hostname : ''; } catch { return ''; }
|
|
})();
|
|
|
|
const bytes = await toAvif(img);
|
|
if (!bytes || bytes.length === 0) {
|
|
failures.push({ source: img?.link || img?.imageUrl || '', reason: 'download or conversion failed' });
|
|
continue;
|
|
}
|
|
|
|
const name = `${slugifyHost(host)}-${results.length + 1}.avif`;
|
|
const relPath = `${IMAGE_FOLDER}/${name}`;
|
|
fs.writeFileSync(path.join(imagesDir, name), bytes);
|
|
|
|
results.push({
|
|
path: relPath,
|
|
domain: host,
|
|
title: typeof img?.title === 'string' ? img.title : '',
|
|
sourcePage: typeof img?.link === 'string' ? img.link : '',
|
|
bytes: bytes.length,
|
|
});
|
|
}
|
|
|
|
if (results.length > 0) {
|
|
try {
|
|
await syncVfsToDb({ googleAuthId, vfsRoot });
|
|
} catch (err) {
|
|
// Non-fatal: the files are on disk and will be picked up by the next
|
|
// javascript_execute / patch sync. Losing the images because a DB write
|
|
// failed would be a worse outcome than a slightly stale store.
|
|
console.warn('[search_images] failed to persist candidates:', err.message);
|
|
}
|
|
}
|
|
|
|
const result = {
|
|
query: term,
|
|
total: results.length,
|
|
returned: results.length,
|
|
images: results,
|
|
nextStep: 'Call analyze_image with these paths (grid) to compare them, then upload the winner.',
|
|
};
|
|
if (failures.length > 0) result.skipped = failures.length;
|
|
if (results.length === 0) {
|
|
result.note = `Found ${raw.length} result(s) for "${term}" but none could be downloaded. Try different wording.`;
|
|
}
|
|
return result;
|
|
},
|
|
});
|
|
} |