u

src/tools/url-get.js

@@ -0,0 +1,107 @@
+import { defineTool, ToolError } from 'libllmorbiter';
+import * as dbCompanySetting from '../db/companysetting.js';
+
+const SERPER_SCRAPE_ENDPOINT = 'https://scrape.serper.dev';
+const SCRAPE_TIMEOUT_MS = 25_000;
+
+/**
+ * Scrapes a web page via Serper Scrape API.
+ *
+ * @param {string} url
+ * @param {string} apiKey
+ * @param {{ signal?: AbortSignal }} [options]
+ * @returns {Promise<object>} raw response from Serper scrape
+ */
+async function serperScrape(url, apiKey, { signal } = {}) {
+    const headers = {
+        'X-API-KEY': apiKey,
+        'Content-Type': 'application/json',
+    };
+
+    const timeoutSignal = AbortSignal.timeout(SCRAPE_TIMEOUT_MS);
+    const combined = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
+
+    let response;
+    try {
+        response = await fetch(SERPER_SCRAPE_ENDPOINT, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({ url, includeMarkdown: true }),
+            redirect: 'follow',
+            signal: combined,
+        });
+    } catch (err) {
+        throw new ToolError(`url_get: network failure contacting Serper scrape (${url}): ${err.message}`);
+    }
+
+    if (!response.ok) {
+        let detail = `${response.status} ${response.statusText}`;
+        try {
+            const errData = await response.json();
+            if (errData?.message) {
+                detail = `${errData.message} (${response.status})`;
+            } else {
+                const text = JSON.stringify(errData);
+                if (text) detail += ` - ${text.slice(0, 300)}`;
+            }
+        } catch {
+            try {
+                const body = await response.text();
+                if (body) detail += ` - ${body.slice(0, 300)}`;
+            } catch { /* ignore */ }
+        }
+        throw new ToolError(`url_get: Serper scrape error: ${detail}`);
+    }
+
+    return await response.json();
+}
+
+/**
+ * Creates the url_get tool.
+ * @param {{ googleAuthId: number }} ctx
+ */
+export function createUrlGetTool({ googleAuthId }) {
+    return defineTool({
+        name: 'url_get',
+        description:
+            'Fetch and scrape the contents of a web page using Serper Scrape. ' +
+            'Returns the raw scrape result as-is from Serper (text, markdown, metadata).',
+        strict: false,
+        parameters: {
+            type: 'object',
+            additionalProperties: false,
+            required: ['url'],
+            properties: {
+                url: {
+                    type: 'string',
+                    description: 'The URL or web address to fetch and scrape (e.g. "https://example.com" or "growheads.de").',
+                },
+            },
+        },
+        invoke: async ({ url } = {}, { signal } = {}) => {
+            const targetUrl = typeof url === 'string' ? url.trim() : '';
+            if (!targetUrl) {
+                throw new ToolError('url_get requires a non-empty "url".');
+            }
+
+            let apiKey = null;
+            try {
+                const setting = await dbCompanySetting.getByKey(googleAuthId, 'keys.serper', { maskWriteonly: false });
+                apiKey = setting?.value?.trim() || null;
+            } catch (err) {
+                console.warn('url_get: failed to read company setting keys.serper:', err?.message || err);
+            }
+            if (!apiKey && process.env.SERPER_API_KEY) {
+                apiKey = process.env.SERPER_API_KEY.trim();
+            }
+
+            if (!apiKey) {
+                throw new ToolError(
+                    'url_get is unavailable: Serper API key not configured (companysetting: keys.serper).'
+                );
+            }
+
+            return await serperScrape(targetUrl, apiKey, { signal });
+        },
+    });
+}