From d74ca40076f3c61dd288923247d485cf68b8fc17 Mon Sep 17 00:00:00 2001 From: seb Date: Tue, 22 Sep 2026 19:11:54 +0200 Subject: [PATCH] u --- src/dir-listing.js | 32 +++++ src/fido.js | 1 + src/tools/index.js | 3 + src/tools/ripgrep.js | 274 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 310 insertions(+) create mode 100644 src/tools/ripgrep.js diff --git a/src/dir-listing.js b/src/dir-listing.js index 16a49cd..7243730 100644 --- a/src/dir-listing.js +++ b/src/dir-listing.js @@ -81,3 +81,35 @@ export function listDir(dir, { seek = 0, limit = 50, recursive = false } = {}) { const truncated = total - (seek + sliced.length); return { entries: sliced, total, truncated: Math.max(0, truncated) }; } + +/** + * Recursively walk the *files* under `dir`, honoring .gitignore rules + * (including nested .gitignore files) plus the hardcoded node_modules rule. + * Returns an array of { abs, rel } where `rel` is relative to ROOT_DIR using + * forward slashes. Symlinks are skipped to avoid loops. + */ +export function walkFiles(dir) { + const results = []; + const walk = (current) => { + // Rebuild per directory so nested .gitignore files apply to their subtree. + const ig = buildIgnoreFilter(current); + let entries; + try { + entries = fs.readdirSync(current, { withFileTypes: true }); + } catch { + return; + } + for (const e of entries) { + const full = path.join(current, e.name); + const rel = path.relative(ROOT_DIR, full).replace(/\\/g, '/'); + if (ig.ignores(e.isDirectory() ? rel + '/' : rel)) continue; + if (e.isDirectory()) { + walk(full); + } else if (e.isFile()) { + results.push({ abs: full, rel }); + } + } + }; + walk(dir); + return results; +} diff --git a/src/fido.js b/src/fido.js index 643372b..6135375 100644 --- a/src/fido.js +++ b/src/fido.js @@ -59,6 +59,7 @@ ${dirSection} ## Available Tools & Guidelines - \`read_file(filename, seek, limit)\`: Inspect file contents and line numbers before modifying them. - \`list_files(dirname, recursive, seek, limit)\`: Explore workspace folders (respects .gitignore). +- \`ripgrep(pattern, path, glob, ignore_case, literal, max_results)\`: Search file contents (regex by default) across the workspace; respects .gitignore and skips binary files. Returns matches with 0-indexed line numbers (usable directly as read_file's seek). - \`patch(operations)\`: Apply file updates or create files. - Create or overwrite: \`{"file": "path", "search": null, "replace": "content"}\` (parent directories auto-created). - Search & replace: \`{"file": "path", "search": "exact old text", "replace": "new text"}\`. Replaces the FIRST match. diff --git a/src/tools/index.js b/src/tools/index.js index a84afd0..4224b85 100644 --- a/src/tools/index.js +++ b/src/tools/index.js @@ -6,6 +6,7 @@ import { listFilesTool } from './list-files.js'; import { readFileTool } from './read-file.js'; +import { ripgrepTool } from './ripgrep.js'; import { patchTool } from './patch.js'; import { mkdirTool } from './mkdir.js'; import { rmdirTool } from './rmdir.js'; @@ -15,6 +16,7 @@ import { askUserTool } from './ask-user.js'; export { listFilesTool, readFileTool, + ripgrepTool, patchTool, mkdirTool, rmdirTool, @@ -25,6 +27,7 @@ export { export const tools = [ listFilesTool, readFileTool, + ripgrepTool, patchTool, mkdirTool, rmdirTool, diff --git a/src/tools/ripgrep.js b/src/tools/ripgrep.js new file mode 100644 index 0000000..22b18e5 --- /dev/null +++ b/src/tools/ripgrep.js @@ -0,0 +1,274 @@ +'use strict'; + +/** + * ripgrep — recursive, .gitignore-aware content search (pure JS, no binaries). + * + * One tool per file: this module owns the search tool plus its small helpers. + * Directory traversal lives in src/dir-listing.js (walkFiles). + */ + +import fs from 'fs'; +import path from 'path'; +import { defineTool, ToolError } from 'llmorbiter'; +import { resolveSafe } from '../sandbox.js'; +import { ROOT_DIR } from '../config.js'; +import { walkFiles } from '../dir-listing.js'; + +// Files larger than this are skipped during a search to keep things responsive. +const MAX_FILE_BYTES = 2 * 1024 * 1024; + +// Longest slice of a matching line echoed back in `text`. Longer lines are capped +// (non-silently: see `textTruncated` on each match and `linesTruncated` in the result). +const MAX_MATCH_TEXT_LEN = 300; + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +/** Escape a string so it can be matched literally inside a RegExp. */ +function escapeRegExp(str) { + return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** + * Translate a simple glob into a RegExp anchored at both ends. + * ** -> any characters (including "/") + * * -> any characters except "/" + * ? -> a single character except "/" + * \\ -> escapes the next character + * All other characters are matched literally (no brace/char-class expansion). + */ +function globToRegExp(pattern) { + let re = ''; + for (let i = 0; i < pattern.length; i++) { + const c = pattern[i]; + if (c === '*') { + if (pattern[i + 1] === '*') { + re += '.*'; + i++; + // Let "**/" also match zero directories. + if (pattern[i + 1] === '/') i++; + } else { + re += '[^/]*'; + } + } else if (c === '?') { + re += '[^/]'; + } else if ('\\^$.|+()[]{}'.includes(c)) { + re += '\\' + c; + } else { + re += c; + } + } + return new RegExp('^' + re + '$'); +} + +/** + * Build a predicate testing a workspace-relative path (forward slashes) against + * a glob. A glob without "/" is also matched against the basename, mirroring + * ripgrep's `-g '*.ext'` behaviour. + */ +function makeGlobMatcher(pattern) { + const rx = globToRegExp(pattern); + const basenameOnly = !pattern.includes('/'); + return (rel) => { + if (rx.test(rel)) return true; + if (basenameOnly) { + const base = rel.slice(rel.lastIndexOf('/') + 1); + return rx.test(base); + } + return false; + }; +} + +/** + * True when a workspace-relative path (forward slashes) lives inside a + * dot-directory — i.e. any *directory* segment starts with ".". Traversal into + * such directories (".git", ".cache", …) is suppressed by default so their + * internals never appear in search results. The final segment is the file name + * and is not checked, so dot-files like ".env" remain searchable. + */ +function isInDotDirectory(rel) { + const segments = rel.split('/'); + for (let i = 0; i < segments.length - 1; i++) { + if (segments[i].startsWith('.')) return true; + } + return false; +} + +// ─── Tool definition ─────────────────────────────────────────────────────────── + +export const ripgrepTool = defineTool({ + name: 'ripgrep', + description: + 'Search file contents across the workspace for a pattern (regex by default). ' + + 'Recurses into directories, honors .gitignore rules, skips binary/large files, ' + + 'and ignores dot-directories (e.g. ".git") by default. ' + + 'Returns matches as { file, line, column, text }, where "line" is 0-indexed so it can be ' + + 'passed straight to read_file\'s seek. Use glob to restrict which files are searched. ' + + 'Long matching lines are capped in "text" (surfaced via each match\'s textTruncated flag ' + + 'and the result\'s linesTruncated count).', + parameters: { + type: 'object', + additionalProperties: false, + required: ['pattern'], + properties: { + pattern: { + type: 'string', + description: 'Regex (or literal string when literal=true) to search for.', + }, + path: { + type: 'string', + description: 'File or directory to search, relative to the workspace root. Default: "."', + }, + glob: { + type: 'string', + description: 'Only search files whose path matches this glob, e.g. "*.js" or "src/**/*.ts".', + }, + ignore_case: { + type: 'boolean', + description: 'Case-insensitive search. Default: false.', + }, + literal: { + type: 'boolean', + description: 'Treat "pattern" as a literal string instead of a regular expression. Default: false.', + }, + max_results: { + type: 'number', + description: 'Maximum number of matches to return. Default: 100.', + }, + }, + }, + invoke: async ({ + pattern, + path: searchPath = '.', + glob, + ignore_case = false, + literal = false, + max_results = 100, + }) => { + if (typeof pattern !== 'string' || pattern.length === 0) { + throw new ToolError('"pattern" must be a non-empty string.'); + } + if (!Number.isInteger(max_results) || max_results < 1) { + throw new ToolError(`"max_results" must be a positive integer (received ${max_results}).`); + } + + const absTarget = resolveSafe(searchPath); + if (!fs.existsSync(absTarget)) { + throw new ToolError(`Path not found: "${searchPath}"`); + } + + // Compile the search pattern once (global so matchAll can iterate hits). + let regex; + try { + regex = new RegExp(literal ? escapeRegExp(pattern) : pattern, ignore_case ? 'gi' : 'g'); + } catch (err) { + throw new ToolError(`Invalid pattern: ${err.message}`); + } + + const globMatch = glob ? makeGlobMatcher(glob) : null; + + // Gather candidate files: a single file, or a recursive walk of a directory. + const targetStat = fs.statSync(absTarget); + const candidates = targetStat.isFile() + ? [{ abs: absTarget, rel: path.relative(ROOT_DIR, absTarget).replace(/\\/g, '/') }] + : walkFiles(absTarget); + + // Drop anything inside a dot-directory (e.g. ".git/…") by default, then apply + // the glob up front so the eligible-file count is known even when the scan + // stops early on max_results. Dot-files (".env", ".gitignore") are kept. + const scoped = candidates.filter( + ({ rel }) => !isInDotDirectory(rel) && (globMatch ? globMatch(rel) : true), + ); + + const matches = []; + let filesSearched = 0; + let filesSkipped = 0; + let linesTruncated = 0; + let truncated = false; + + search: + for (const { abs, rel } of scoped) { + let size = 0; + try { + size = fs.statSync(abs).size; + } catch { + continue; + } + if (size > MAX_FILE_BYTES) { + filesSkipped++; + continue; + } + + let content; + try { + content = fs.readFileSync(abs, 'utf8'); + } catch { + continue; + } + // Skip binary files (a NUL byte is a good heuristic). + if (content.includes('\u0000')) { + filesSkipped++; + continue; + } + + filesSearched++; + const lines = content.split('\n'); + for (let i = 0; i < lines.length; i++) { + const rawLine = lines[i]; + const lineText = rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine; + let shown; + let capped = false; + for (const m of lineText.matchAll(regex)) { + if (matches.length >= max_results) { + // We found one more than requested: stop and flag truncation. + truncated = true; + break search; + } + // Cap long lines once per line, only when a match actually needs it. + if (shown === undefined) { + capped = lineText.length > MAX_MATCH_TEXT_LEN; + shown = capped ? lineText.slice(0, MAX_MATCH_TEXT_LEN) + '…' : lineText; + } + const entry = { file: rel, line: i, column: m.index, text: shown }; + if (capped) { + entry.textTruncated = true; + entry.lineLength = lineText.length; + } + matches.push(entry); + } + if (capped) linesTruncated++; + } + } + + const result = { + pattern, + path: searchPath, + glob: glob ?? null, + ignoreCase: ignore_case, + literal, + filesSearched, + filesEligible: scoped.length, + matches, + returned: matches.length, + }; + if (filesSkipped > 0) { + result.skippedFiles = filesSkipped; + result.skippedNote = `${filesSkipped} file(s) skipped (binary or larger than ${MAX_FILE_BYTES} bytes).`; + } + if (linesTruncated > 0) { + result.linesTruncated = linesTruncated; + result.textTruncationNote = + `${linesTruncated} matching line(s) longer than ${MAX_MATCH_TEXT_LEN} chars were capped in "text" ` + + `(marked with a trailing "…" and textTruncated: true; "column" still refers to the full line).`; + } + if (truncated) { + result.truncated = true; + result.truncationNote = + `Showing the first ${max_results} matches. Refine the pattern or raise max_results to see more. ` + + `(Search stopped early: filesSearched counts only files scanned so far; ` + + `${scoped.length} file(s) were eligible before scanning.)`; + } else { + result.totalMatches = matches.length; + } + return result; + }, +});