import { defineTool, ToolError } from 'libllmorbiter'; import * as dbCompanySetting from '../db/companysetting.js'; const SERPER_SCRAPE_ENDPOINT = 'https://scrape.serper.dev'; const SCRAPE_TIMEOUT_MS = 25_000; /** * Scrapes a web page via Serper Scrape API. * * @param {string} url * @param {string} apiKey * @param {{ signal?: AbortSignal }} [options] * @returns {Promise} raw response from Serper scrape */ async function serperScrape(url, apiKey, { signal } = {}) { const headers = { 'X-API-KEY': apiKey, 'Content-Type': 'application/json', }; const timeoutSignal = AbortSignal.timeout(SCRAPE_TIMEOUT_MS); const combined = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal; let response; try { response = await fetch(SERPER_SCRAPE_ENDPOINT, { method: 'POST', headers, body: JSON.stringify({ url, includeMarkdown: true }), redirect: 'follow', signal: combined, }); } catch (err) { throw new ToolError(`url_get: network failure contacting Serper scrape (${url}): ${err.message}`); } if (!response.ok) { let detail = `${response.status} ${response.statusText}`; try { const errData = await response.json(); if (errData?.message) { detail = `${errData.message} (${response.status})`; } else { const text = JSON.stringify(errData); if (text) detail += ` - ${text.slice(0, 300)}`; } } catch { try { const body = await response.text(); if (body) detail += ` - ${body.slice(0, 300)}`; } catch { /* ignore */ } } throw new ToolError(`url_get: Serper scrape error: ${detail}`); } return await response.json(); } /** * Creates the url_get tool. * @param {{ googleAuthId: number }} ctx */ export function createUrlGetTool({ googleAuthId }) { return defineTool({ name: 'url_get', description: 'Fetch and scrape web page content via Serper Scrape (text, markdown, metadata).', strict: false, parameters: { type: 'object', additionalProperties: false, required: ['url'], properties: { url: { type: 'string', description: 'Target URL.', }, }, }, invoke: async ({ url } = {}, { signal } = {}) => { const targetUrl = typeof url === 'string' ? url.trim() : ''; if (!targetUrl) { throw new ToolError('url_get requires a non-empty "url".'); } let apiKey = null; try { const setting = await dbCompanySetting.getByKey(googleAuthId, 'keys.serper', { maskWriteonly: false }); apiKey = setting?.value?.trim() || null; } catch (err) { console.warn('url_get: failed to read company setting keys.serper:', err?.message || err); } if (!apiKey && process.env.SERPER_API_KEY) { apiKey = process.env.SERPER_API_KEY.trim(); } if (!apiKey) { throw new ToolError( 'url_get is unavailable: Serper API key not configured (companysetting: keys.serper).' ); } return await serperScrape(targetUrl, apiKey, { signal }); }, }); }