106 lines
3.5 KiB
JavaScript
106 lines
3.5 KiB
JavaScript
import { defineTool, ToolError } from 'libllmorbiter';
|
|
import * as dbCompanySetting from '../db/companysetting.js';
|
|
|
|
const SERPER_SCRAPE_ENDPOINT = 'https://scrape.serper.dev';
|
|
const SCRAPE_TIMEOUT_MS = 25_000;
|
|
|
|
/**
|
|
* Scrapes a web page via Serper Scrape API.
|
|
*
|
|
* @param {string} url
|
|
* @param {string} apiKey
|
|
* @param {{ signal?: AbortSignal }} [options]
|
|
* @returns {Promise<object>} raw response from Serper scrape
|
|
*/
|
|
async function serperScrape(url, apiKey, { signal } = {}) {
|
|
const headers = {
|
|
'X-API-KEY': apiKey,
|
|
'Content-Type': 'application/json',
|
|
};
|
|
|
|
const timeoutSignal = AbortSignal.timeout(SCRAPE_TIMEOUT_MS);
|
|
const combined = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
|
|
|
let response;
|
|
try {
|
|
response = await fetch(SERPER_SCRAPE_ENDPOINT, {
|
|
method: 'POST',
|
|
headers,
|
|
body: JSON.stringify({ url, includeMarkdown: true }),
|
|
redirect: 'follow',
|
|
signal: combined,
|
|
});
|
|
} catch (err) {
|
|
throw new ToolError(`url_get: network failure contacting Serper scrape (${url}): ${err.message}`);
|
|
}
|
|
|
|
if (!response.ok) {
|
|
let detail = `${response.status} ${response.statusText}`;
|
|
try {
|
|
const errData = await response.json();
|
|
if (errData?.message) {
|
|
detail = `${errData.message} (${response.status})`;
|
|
} else {
|
|
const text = JSON.stringify(errData);
|
|
if (text) detail += ` - ${text.slice(0, 300)}`;
|
|
}
|
|
} catch {
|
|
try {
|
|
const body = await response.text();
|
|
if (body) detail += ` - ${body.slice(0, 300)}`;
|
|
} catch { /* ignore */ }
|
|
}
|
|
throw new ToolError(`url_get: Serper scrape error: ${detail}`);
|
|
}
|
|
|
|
return await response.json();
|
|
}
|
|
|
|
/**
|
|
* Creates the url_get tool.
|
|
* @param {{ googleAuthId: number }} ctx
|
|
*/
|
|
export function createUrlGetTool({ googleAuthId }) {
|
|
return defineTool({
|
|
name: 'url_get',
|
|
description: 'Fetch and scrape web page content via Serper Scrape (text, markdown, metadata).',
|
|
strict: false,
|
|
parameters: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
required: ['url'],
|
|
properties: {
|
|
url: {
|
|
type: 'string',
|
|
description: 'Target URL.',
|
|
},
|
|
},
|
|
},
|
|
invoke: async ({ url } = {}, { signal } = {}) => {
|
|
const targetUrl = typeof url === 'string' ? url.trim() : '';
|
|
if (!targetUrl) {
|
|
throw new ToolError('url_get requires a non-empty "url".');
|
|
}
|
|
|
|
let apiKey = null;
|
|
try {
|
|
const setting = await dbCompanySetting.getByKey(googleAuthId, 'keys.serper', { maskWriteonly: false });
|
|
apiKey = setting?.value?.trim() || null;
|
|
} catch (err) {
|
|
console.warn('url_get: failed to read company setting keys.serper:', err?.message || err);
|
|
}
|
|
if (!apiKey && process.env.SERPER_API_KEY) {
|
|
apiKey = process.env.SERPER_API_KEY.trim();
|
|
}
|
|
|
|
if (!apiKey) {
|
|
throw new ToolError(
|
|
'url_get is unavailable: Serper API key not configured (companysetting: keys.serper).'
|
|
);
|
|
}
|
|
|
|
return await serperScrape(targetUrl, apiKey, { signal });
|
|
},
|
|
});
|
|
}
|