diff --git a/package-lock.json b/package-lock.json index 796f8a7..afe275d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "libdashstore": "file:../dashdbschema", "libllmorbiter": "file:../llmorbiter", "multer": "^2.4.0", + "pdfconvert-wasm": "^0.3.0", "react": "^19.3.0", "react-canvas-label-editor": "github:sebseb7/react-canvas-label-editor", "selfsigned": "^5.5.0", @@ -3182,6 +3183,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/pdfconvert-wasm": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/pdfconvert-wasm/-/pdfconvert-wasm-0.3.0.tgz", + "integrity": "sha512-FbEk+Grg0UST4kOPuN7OFfVsut1FvDOJNPS3pt4rzCS8VlFsFzD7ptg7YtFZjEOKgJ4MgqlfCgjK4RYaRBpT0Q==", + "license": "GPLv3" + }, "node_modules/pkijs": { "version": "3.4.1", "resolved": "https://registry.npmjs.org/pkijs/-/pkijs-3.4.1.tgz", diff --git a/package.json b/package.json index 9409a21..4890cfd 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "libdashstore": "file:../dashdbschema", "libllmorbiter": "file:../llmorbiter", "multer": "^2.4.0", + "pdfconvert-wasm": "^0.3.0", "react": "^19.3.0", "react-canvas-label-editor": "github:sebseb7/react-canvas-label-editor", "selfsigned": "^5.5.0", diff --git a/src/services/conversationManager.js b/src/services/conversationManager.js index 0ae4d76..5045072 100644 --- a/src/services/conversationManager.js +++ b/src/services/conversationManager.js @@ -20,6 +20,9 @@ import { createPatchTool } from '../tools/patch.js'; import { createRmTool } from '../tools/rm.js'; import { createSearchImagesTool } from '../tools/search-images.js'; import { createAnalyzeImageTool } from '../tools/analyze-image.js'; +import { createWebSearchTool } from '../tools/web-search.js'; +import { createUrlGetTool } from '../tools/url-get.js'; +import { createPdfReadTool } from '../tools/pdf-read.js'; import { resolveOpenRouterKey, resolveDefaultModel, RETIRED_MODELS, hasConfiguredOpenRouterKey } from '../tools/llm-key.js'; const KEEP_ALIVE_MS = 20_000; @@ -114,9 +117,12 @@ async function buildSystemPrompt(vfsRoot, googleAuthId) { '6. patch: Create, overwrite, or search-and-replace text in a VFS file. This is the only text write path — it also creates SKILL files.', '7. search_images: Search Google Images and store the candidates in the VFS as AVIF files.', '8. analyze_image: Have a vision model look at up to 9 images (VFS paths or "image/") and return its judgement. This is how you "see" — never claim to have inspected an image yourself.', - '9. reset_vfs: Delete files, images, and artifacts stored in the virtual filesystem (VFS) to free up storage or clean up. SKILL files are exempt and survive.', - '10. rm: Delete a single file from the VFS. This is how a SKILL file is deleted on purpose, since reset_vfs always spares skills.', - '11. apisuggestionsorissues: Report missing API endpoints, bugs, or issues to be triaged. Automatically writes an entry to the company history table for review.', + '9. web_search: Search the web using Google Search. Returns organic results, snippets, and knowledge graph. Supports time filters via `tbs` ("qdr:d" past day, "qdr:w" past week, "qdr:m" past month, "qdr:y" past year).', + '10. url_get: Fetch and scrape the content and markdown of a web page/URL.', + '11. pdf_read: Read and convert PDF or text files. Converts PDF to layout-preserved text ("txt") or renders pages into PNG images ("png") saved in the VFS images/ folder.', + '12. reset_vfs: Delete files, images, and artifacts stored in the virtual filesystem (VFS) to free up storage or clean up. SKILL files are exempt and survive.', + '13. rm: Delete a single file from the VFS. This is how a SKILL file is deleted on purpose, since reset_vfs always spares skills.', + '14. apisuggestionsorissues: Report missing API endpoints, bugs, or issues to be triaged. Automatically writes an entry to the company history table for review.', '', 'VIRTUAL FILESYSTEM:', '- You have a persistent per-company VFS. Paths are relative to the VFS root and always use forward slashes, e.g. "labels/sheet1.png".', @@ -589,6 +595,9 @@ export class ConversationManager { createRmTool({ googleAuthId }), createSearchImagesTool({ googleAuthId }), createAnalyzeImageTool({ googleAuthId, model }), + createWebSearchTool({ googleAuthId }), + createUrlGetTool({ googleAuthId }), + createPdfReadTool({ googleAuthId }), createResetVfsTool({ googleAuthId }), createApiSuggestionsOrIssuesTool({ googleAuthId, companyId: this.companyId }), ], diff --git a/src/tools/pdf-read.js b/src/tools/pdf-read.js new file mode 100644 index 0000000..2bb2b7c --- /dev/null +++ b/src/tools/pdf-read.js @@ -0,0 +1,214 @@ +import fs from 'fs'; +import path from 'path'; +import { defineTool, ToolError } from 'libllmorbiter'; +import { pdfToText, splitPdfPages, pdfToPng } from 'pdfconvert-wasm'; +import { prepareVfs, syncVfsToDb } from './vfs.js'; + +const IMAGE_FOLDER = 'images'; +const DEFAULT_DPI = 150; + +/** + * Normalizes format input to 'txt', 'png', or 'both'. + * @param {string} [format] + * @returns {'txt'|'png'|'both'} + */ +function normalizeFormat(format) { + if (!format || typeof format !== 'string') return 'txt'; + const f = format.trim().toLowerCase(); + if (f === 'png' || f === 'image' || f === 'images') return 'png'; + if (f === 'both' || f === 'all') return 'both'; + return 'txt'; +} + +/** + * Slugifies a filename base for safe image output names. + * @param {string} filename + * @returns {string} + */ +function slugifyBase(filename) { + const base = path.basename(filename, path.extname(filename)); + const cleaned = String(base || '') + .toLowerCase() + .replace(/[^a-z0-9.]+/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, ''); + return cleaned.slice(0, 40) || 'document'; +} + +/** + * Creates the pdf_read tool. + * @param {{ googleAuthId: number }} ctx + */ +export function createPdfReadTool({ googleAuthId }) { + return defineTool({ + name: 'pdf_read', + description: + 'Read and convert PDF (or text) documents. ' + + 'Supports converting PDF to layout-preserved text ("txt") or rendering pages to PNG images ("png") ' + + 'stored in the VFS images/ folder for inspection with analyze_image or inline markdown embedding.', + strict: false, + parameters: { + type: 'object', + additionalProperties: false, + required: ['path'], + properties: { + path: { + type: 'string', + description: 'VFS path of the PDF or text file (e.g. "invoice.pdf", "docs/order.pdf"), or HTTP(S) URL.', + }, + format: { + type: 'string', + enum: ['txt', 'png', 'both'], + description: + 'Conversion format: "txt" extracts layout-preserved text, ' + + '"png" renders pages into PNG images stored in the VFS images/ folder, ' + + '"both" extracts text and renders PNG images. Default is "txt".', + }, + page: { + type: 'number', + description: 'Specific page number (1-based) to extract text or render as image. If omitted, all pages are processed.', + }, + dpi: { + type: 'number', + description: 'Resolution in DPI for PNG rendering. Default 150.', + }, + }, + }, + invoke: async ({ path: inputPath, file, format, type, page, dpi = DEFAULT_DPI } = {}) => { + const rawPath = typeof inputPath === 'string' ? inputPath.trim() : typeof file === 'string' ? file.trim() : ''; + if (!rawPath) { + throw new ToolError('pdf_read requires a non-empty "path" (e.g. "document.pdf").'); + } + + const targetFormat = normalizeFormat(format || type); + const targetPage = Number.isInteger(Number(page)) && Number(page) >= 1 ? Number(page) : null; + const targetDpi = Number.isInteger(Number(dpi)) && Number(dpi) >= 50 && Number(dpi) <= 600 ? Number(dpi) : DEFAULT_DPI; + + const vfsRoot = await prepareVfs({ googleAuthId }); + let buffer = null; + let displayPath = rawPath; + + if (rawPath.startsWith('http://') || rawPath.startsWith('https://')) { + try { + const res = await fetch(rawPath, { signal: AbortSignal.timeout(15_000) }); + if (!res.ok) { + throw new Error(`${res.status} ${res.statusText}`); + } + buffer = Buffer.from(await res.arrayBuffer()); + } catch (err) { + throw new ToolError(`pdf_read: network failure downloading URL (${rawPath}): ${err.message}`); + } + } else { + const clean = rawPath.replace(/^([a-zA-Z]:|\/)+/, '').replace(/^\.\/+/, ''); + const abs = path.resolve(vfsRoot, clean); + const rel = path.relative(vfsRoot, abs); + if (rel.startsWith('..') || path.isAbsolute(rel)) { + throw new ToolError(`Access denied: path "${rawPath}" escapes virtual filesystem boundary.`); + } + displayPath = rel; + + if (!fs.existsSync(abs) || !fs.statSync(abs).isFile()) { + throw new ToolError(`File not found: "${rawPath}". Check the path with find_files.`); + } + buffer = fs.readFileSync(abs); + } + + if (!buffer || buffer.length === 0) { + throw new ToolError(`File is empty: "${rawPath}".`); + } + + const isPdf = buffer.subarray(0, 5).toString('ascii').startsWith('%PDF') || rawPath.toLowerCase().endsWith('.pdf'); + + // Plain text fallback if non-PDF + if (!isPdf) { + const text = buffer.toString('utf-8'); + return { + path: displayPath, + format: 'txt', + totalPages: 1, + text, + }; + } + + // PDF processing via pdfconvert-wasm + // Always clone as isolated Uint8Array so WASM worker does not detach shared buffers. + const pdfBytes = Uint8Array.from(buffer); + + const result = { + path: displayPath, + format: targetFormat, + }; + + // 1. Text conversion + if (targetFormat === 'txt' || targetFormat === 'both') { + try { + if (targetPage) { + const pageText = await pdfToText(pdfBytes, { + layout: true, + firstPage: targetPage, + lastPage: targetPage, + }); + result.page = targetPage; + result.text = pageText; + } else { + const text = await pdfToText(pdfBytes, { layout: true }); + const pages = splitPdfPages(text); + result.totalPages = pages.length; + result.text = text; + result.pages = pages.map((t, idx) => ({ pageNumber: idx + 1, text: t })); + } + } catch (err) { + throw new ToolError(`pdf_read: failed to extract text from PDF: ${err.message}`); + } + } + + // 2. PNG conversion + if (targetFormat === 'png' || targetFormat === 'both') { + try { + const options = { dpi: targetDpi }; + if (targetPage) { + options.firstPage = targetPage; + options.lastPage = targetPage; + } + + const images = await pdfToPng(Uint8Array.from(buffer), options); + const imagesDir = path.join(vfsRoot, IMAGE_FOLDER); + fs.mkdirSync(imagesDir, { recursive: true }); + + const baseSlug = slugifyBase(displayPath); + const savedImages = []; + + for (const img of images) { + const filename = `${baseSlug}-page-${img.pageNumber}.png`; + const absImgPath = path.join(imagesDir, filename); + const relImgPath = `${IMAGE_FOLDER}/${filename}`; + + fs.writeFileSync(absImgPath, Buffer.from(img.data)); + savedImages.push({ + pageNumber: img.pageNumber, + path: relImgPath, + bytes: img.data.length, + }); + } + + if (savedImages.length > 0) { + try { + await syncVfsToDb({ googleAuthId, vfsRoot }); + } catch (err) { + console.warn('[pdf_read] failed to sync rendered images to VFS DB:', err?.message || err); + } + } + + result.images = savedImages; + if (!result.totalPages) result.totalPages = images.length; + result.note = + 'Rendered page PNGs saved to VFS. Use analyze_image to inspect them or embed them with markdown `![Page](path)`.'; + } catch (err) { + throw new ToolError(`pdf_read: failed to render PDF to PNG: ${err.message}`); + } + } + + return result; + }, + }); +} diff --git a/src/tools/url-get.js b/src/tools/url-get.js new file mode 100644 index 0000000..5ad63e3 --- /dev/null +++ b/src/tools/url-get.js @@ -0,0 +1,107 @@ +import { defineTool, ToolError } from 'libllmorbiter'; +import * as dbCompanySetting from '../db/companysetting.js'; + +const SERPER_SCRAPE_ENDPOINT = 'https://scrape.serper.dev'; +const SCRAPE_TIMEOUT_MS = 25_000; + +/** + * Scrapes a web page via Serper Scrape API. + * + * @param {string} url + * @param {string} apiKey + * @param {{ signal?: AbortSignal }} [options] + * @returns {Promise} raw response from Serper scrape + */ +async function serperScrape(url, apiKey, { signal } = {}) { + const headers = { + 'X-API-KEY': apiKey, + 'Content-Type': 'application/json', + }; + + const timeoutSignal = AbortSignal.timeout(SCRAPE_TIMEOUT_MS); + const combined = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal; + + let response; + try { + response = await fetch(SERPER_SCRAPE_ENDPOINT, { + method: 'POST', + headers, + body: JSON.stringify({ url, includeMarkdown: true }), + redirect: 'follow', + signal: combined, + }); + } catch (err) { + throw new ToolError(`url_get: network failure contacting Serper scrape (${url}): ${err.message}`); + } + + if (!response.ok) { + let detail = `${response.status} ${response.statusText}`; + try { + const errData = await response.json(); + if (errData?.message) { + detail = `${errData.message} (${response.status})`; + } else { + const text = JSON.stringify(errData); + if (text) detail += ` - ${text.slice(0, 300)}`; + } + } catch { + try { + const body = await response.text(); + if (body) detail += ` - ${body.slice(0, 300)}`; + } catch { /* ignore */ } + } + throw new ToolError(`url_get: Serper scrape error: ${detail}`); + } + + return await response.json(); +} + +/** + * Creates the url_get tool. + * @param {{ googleAuthId: number }} ctx + */ +export function createUrlGetTool({ googleAuthId }) { + return defineTool({ + name: 'url_get', + description: + 'Fetch and scrape the contents of a web page using Serper Scrape. ' + + 'Returns the raw scrape result as-is from Serper (text, markdown, metadata).', + strict: false, + parameters: { + type: 'object', + additionalProperties: false, + required: ['url'], + properties: { + url: { + type: 'string', + description: 'The URL or web address to fetch and scrape (e.g. "https://example.com" or "growheads.de").', + }, + }, + }, + invoke: async ({ url } = {}, { signal } = {}) => { + const targetUrl = typeof url === 'string' ? url.trim() : ''; + if (!targetUrl) { + throw new ToolError('url_get requires a non-empty "url".'); + } + + let apiKey = null; + try { + const setting = await dbCompanySetting.getByKey(googleAuthId, 'keys.serper', { maskWriteonly: false }); + apiKey = setting?.value?.trim() || null; + } catch (err) { + console.warn('url_get: failed to read company setting keys.serper:', err?.message || err); + } + if (!apiKey && process.env.SERPER_API_KEY) { + apiKey = process.env.SERPER_API_KEY.trim(); + } + + if (!apiKey) { + throw new ToolError( + 'url_get is unavailable: Serper API key not configured (companysetting: keys.serper).' + ); + } + + return await serperScrape(targetUrl, apiKey, { signal }); + }, + }); +} diff --git a/src/tools/web-search.js b/src/tools/web-search.js new file mode 100644 index 0000000..9027f9b --- /dev/null +++ b/src/tools/web-search.js @@ -0,0 +1,165 @@ +import { defineTool, ToolError } from 'libllmorbiter'; +import * as dbCompanySetting from '../db/companysetting.js'; + +const SERPER_SEARCH_ENDPOINT = 'https://google.serper.dev/search'; +const SEARCH_TIMEOUT_MS = 15_000; + +/** + * Normalizes user/model-supplied time filter (`tbs`) into Google's qdr syntax. + * Supported: + * - "qdr:d" or "d" / "day" / "24h" -> "qdr:d" + * - "qdr:w" or "w" / "week" -> "qdr:w" + * - "qdr:m" or "m" / "month" -> "qdr:m" + * - "qdr:y" or "y" / "year" -> "qdr:y" + * - "qdr:h" or "h" / "hour" -> "qdr:h" + * + * @param {string|null} tbs + * @returns {string|undefined} + */ +function normalizeTbs(tbs) { + if (!tbs || typeof tbs !== 'string') return undefined; + const trimmed = tbs.trim().toLowerCase(); + if (!trimmed) return undefined; + + if (trimmed.startsWith('qdr:')) return trimmed; + if (trimmed === 'd' || trimmed === 'day' || trimmed === '24h') return 'qdr:d'; + if (trimmed === 'w' || trimmed === 'week') return 'qdr:w'; + if (trimmed === 'm' || trimmed === 'month') return 'qdr:m'; + if (trimmed === 'y' || trimmed === 'year') return 'qdr:y'; + if (trimmed === 'h' || trimmed === 'hour') return 'qdr:h'; + + return trimmed; +} + +/** + * Queries Serper for Google web search results. + * + * @param {string} query + * @param {string} apiKey + * @param {{ gl?: string, hl?: string, tbs?: string, page?: number, signal?: AbortSignal }} [options] + * @returns {Promise} raw response from Serper + */ +async function serperWebSearch(query, apiKey, { gl = 'de', hl = 'de', tbs, page = 1, signal } = {}) { + const headers = { + 'X-API-KEY': apiKey, + 'Content-Type': 'application/json', + }; + + const timeoutSignal = AbortSignal.timeout(SEARCH_TIMEOUT_MS); + const combined = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal; + + const payload = { + q: query, + gl, + hl, + }; + if (tbs) payload.tbs = tbs; + if (page && page > 1) payload.page = page; + + let response; + try { + response = await fetch(SERPER_SEARCH_ENDPOINT, { + method: 'POST', + headers, + body: JSON.stringify(payload), + redirect: 'follow', + signal: combined, + }); + } catch (err) { + throw new ToolError(`web_search: network failure contacting Serper (${query}): ${err.message}`); + } + + if (!response.ok) { + let detail = `${response.status} ${response.statusText}`; + try { + const body = await response.text(); + if (body) detail += ` - ${body.slice(0, 300)}`; + } catch { /* body already consumed or unreadable */ } + throw new ToolError(`web_search: Serper API error: ${detail}`); + } + + return await response.json(); +} + +/** + * Creates the web_search tool. + * @param {{ googleAuthId: number }} ctx + */ +export function createWebSearchTool({ googleAuthId }) { + return defineTool({ + name: 'web_search', + description: + 'Search the web using Google Search via Serper.dev. ' + + 'Returns raw search results as-is from Serper. ' + + 'Supports time filters via `tbs` ("qdr:d" for past day, "qdr:w" for past week, ' + + '"qdr:m" for past month, "qdr:y" for past year). Each page returns 10 results.', + strict: false, + parameters: { + type: 'object', + additionalProperties: false, + required: ['query'], + properties: { + query: { + type: 'string', + description: 'Search query, e.g. "apple inc" or "vite vue3 release".', + }, + tbs: { + type: 'string', + description: + 'Time filter for results: "qdr:d" (past 24h / day), "qdr:w" (past week), ' + + '"qdr:m" (past month), "qdr:y" (past year). Short forms like "d", "w", "m", "y" are also accepted.', + }, + gl: { + type: 'string', + description: 'Geolocation country code. Default "de".', + }, + hl: { + type: 'string', + description: 'Result language code. Default "de".', + }, + page: { + type: 'number', + description: 'Page number of results. Default 1.', + }, + }, + }, + invoke: async ({ query, tbs, gl = 'de', hl = 'de', page = 1 } = {}, { signal } = {}) => { + const term = typeof query === 'string' ? query.trim() : ''; + if (!term) { + throw new ToolError('web_search requires a non-empty "query".'); + } + + const parsedPage = Number(page); + if (!Number.isInteger(parsedPage) || parsedPage < 1) { + throw new ToolError(`"page" must be a positive integer (received ${page}).`); + } + + const normalizedTbs = normalizeTbs(tbs); + + let apiKey = null; + try { + const setting = await dbCompanySetting.getByKey(googleAuthId, 'keys.serper', { maskWriteonly: false }); + apiKey = setting?.value?.trim() || null; + } catch (err) { + console.warn('web_search: failed to read company setting keys.serper:', err?.message || err); + } + if (!apiKey && process.env.SERPER_API_KEY) { + apiKey = process.env.SERPER_API_KEY.trim(); + } + + if (!apiKey) { + throw new ToolError( + 'web_search is unavailable: Serper API key not configured (companysetting: keys.serper).' + ); + } + + return await serperWebSearch(term, apiKey, { + gl, + hl, + tbs: normalizedTbs, + page: parsedPage, + signal, + }); + }, + }); +}