llmorbiter/examples/tool-error-retry.js

'use strict';

/**
 * Example & Test: Tool error handling and LLM self-correction
 *
 * Demonstrates:
 *   1. A tool throwing an Error (e.g. invalid parameter format).
 *   2. LLMOrbiter catching the error, emitting a 'tool_error' event, and
 *      formatting the error back to the LLM in the tool message.
 *   3. The LLM inspecting the error feedback and retrying the tool call with corrected arguments.
 *   4. Successful run completion.
 *
 * Usage:
 *   node examples/tool-error-retry.js
 *   node examples/tool-error-retry.js --model deepseek/deepseek-v4-flash-0731
 */

import path from 'path';
import { fileURLToPath } from 'url';
import dotenv from 'dotenv';

import { LLMOrbiter, OpenRouterProvider, defineTool, ToolError } from '../src/index.js';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
dotenv.config({ path: path.resolve(__dirname, '.env') });

const args = process.argv.slice(2);
function getArg(flag) {
    const idx = args.indexOf(flag);
    return idx !== -1 && args[idx + 1] ? args[idx + 1] : null;
}

const apiKey = getArg('--key') || process.env.OPENROUTER_API_KEY;
const model = getArg('--model') || 'deepseek/deepseek-v4-flash-0731';

if (!apiKey) {
    console.error('Error: Missing API key. Set OPENROUTER_API_KEY in examples/.env or pass --key <api-key>');
    process.exit(1);
}

// ─── Tool with error throwing ───────────────────────────────────────────────────

let callCount = 0;

const orderLookupTool = defineTool({
    name: 'lookup_order',
    description: 'Look up an order by order ID.',
    parameters: {
        type: 'object',
        required: ['orderId'],
        additionalProperties: false,
        properties: {
            orderId: {
                type: 'string',
                description: 'The order ID to check',
            },
        },
    },
    invoke: async ({ orderId }) => {
        callCount++;
        // Enforce the requirement that order IDs must start with "ORD-"
        if (!orderId.startsWith('ORD-')) {
            throw new ToolError(
                `Invalid order ID "${orderId}". Order IDs must start with the prefix "ORD-", e.g. "ORD-${orderId}". Please correct the ID and retry.`
            );
        }

        return {
            orderId,
            status: 'Delivered',
            item: 'Mechanical Keyboard',
            trackingNumber: 'TRK-98765432',
        };
    },
});

// ─── Main ───────────────────────────────────────────────────────────────────────

async function main() {
    console.log('🧪 Starting Tool Error & Retry Test...\n');

    const provider = new OpenRouterProvider({ apiKey });
    const events = [];

    const orbiter = new LLMOrbiter(provider, {
        onStatus: (status) => {
            events.push({ type: 'status', data: status });
            console.log(`⚡ ${status}`);
        },
        onToolCall: (call) => {
            events.push({ type: 'tool_call', data: call });
            console.log(`🔧 Tool call: ${call.name}(${JSON.stringify(call.args)})`);
        },
        onToolError: (err) => {
            events.push({ type: 'tool_error', data: err });
            console.log(`❌ Tool error:`, err.error);
        },
        onToolResult: (res) => {
            events.push({ type: 'tool_result', data: res });
            console.log(`📦 Tool result:`, res.result);
        },
        onUsage: (usage) => {
            events.push({ type: 'usage', data: usage });
            console.log(`📊 LLM call usage:`, usage);
        },
    });

    const result = await orbiter.run({
        prompt: 'Please check the status of order 54321. If the tool reports an error, fix the ID according to the error message and call the tool again.',
        model,
        tools: [orderLookupTool],
    });

    console.log('\n💬 Final Response:\n', result.content);

    // ─── Test Assertions ─────────────────────────────────────────────────────────
    console.log('\n─── Test Results ───');

    const errorEvent = events.find((e) => e.type === 'tool_error');
    const resultEvent = events.find((e) => e.type === 'tool_result');
    const toolCalls = events.filter((e) => e.type === 'tool_call');

    const hasError = !!errorEvent;
    const hasRetry = toolCalls.length >= 2;
    const hasResult = !!resultEvent;
    const hasContent = !!result.content;

    console.log(hasError ? '✅ PASS: Tool error was thrown and caught' : '❌ FAIL: Tool error was not emitted');
    console.log(hasRetry ? '✅ PASS: LLM called the tool again with correction' : '❌ FAIL: Tool was not called again');
    console.log(hasResult ? '✅ PASS: Corrected tool call succeeded' : '❌ FAIL: Tool never succeeded');
    console.log(hasContent ? '✅ PASS: LLM generated final answer from result' : '❌ FAIL: No final response');

    if (hasError && hasRetry && hasResult && hasContent) {
        console.log('\n🎉 ALL TESTS PASSED: Tool error and retry workflow verified successfully!\n');
    } else {
        console.error('\n⚠️ Some assertions failed.\n');
        process.exitCode = 1;
    }
}

main().catch(console.error);