From d2920fd39a055f74242c39a2fa386b8addb603ba Mon Sep 17 00:00:00 2001 From: sebseb7 Date: Sat, 4 Apr 2026 21:51:52 +0200 Subject: [PATCH] token count for search --- package-lock.json | 9 +- package.json | 3 +- public/index.html | 211 ++++++++++++++++++++++++++++++ searchCLI.js | 3 - src/routes/search.js | 4 +- src/services/openRouterLogger.js | 4 +- src/services/openRouterService.js | 105 ++++++++++++++- src/services/searchFormatter.js | 9 +- src/services/searchService.js | 151 +++++++++++++++++++-- src/utils/htmlConsoleRenderer.js | 8 ++ 10 files changed, 477 insertions(+), 30 deletions(-) diff --git a/package-lock.json b/package-lock.json index cfd8b04..996283e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,8 @@ "chalk": "^5.6.2", "dotenv": "^17.2.3", "exa-js": "^2.11.0", - "express": "^5.2.1" + "express": "^5.2.1", + "tiktoken": "^1.0.22" } }, "node_modules/@openrouter/sdk": { @@ -889,6 +890,12 @@ "node": ">= 0.8" } }, + "node_modules/tiktoken": { + "version": "1.0.22", + "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.22.tgz", + "integrity": "sha512-PKvy1rVF1RibfF3JlXBSP0Jrcw2uq3yXdgcEXtKTYn3QJ/cBRBHDnrJ5jHky+MENZ6DIPwNUGWpkVx+7joCpNA==", + "license": "MIT" + }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", diff --git a/package.json b/package.json index d05db90..d00d392 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "chalk": "^5.6.2", "dotenv": "^17.2.3", "exa-js": "^2.11.0", - "express": "^5.2.1" + "express": "^5.2.1", + "tiktoken": "^1.0.22" } } diff --git a/public/index.html b/public/index.html index 463e00c..08420fa 100644 --- a/public/index.html +++ b/public/index.html @@ -289,6 +289,67 @@ text-decoration: underline; } + /* Cost Breakdown Styles */ + .cost-breakdown-details { + margin-top: 10px; + } + + .cost-breakdown-summary { + cursor: pointer; + font-weight: 600; + color: #555; + padding: 12px 15px; + background: #f8f9fa; + border-radius: 6px; + border-left: 4px solid #667eea; + transition: background 0.2s; + list-style: none; + } + + .cost-breakdown-summary:hover { + background: #e9ecef; + } + + .cost-breakdown-summary::-webkit-details-marker { + display: none; + } + + .cost-breakdown-content { + padding: 15px; + margin-top: 10px; + background: #fff; + border: 1px solid #e0e0e0; + border-radius: 6px; + } + + .cost-table { + width: 100%; + border-collapse: collapse; + font-size: 0.9rem; + } + + .cost-table th, + .cost-table td { + padding: 10px 12px; + text-align: left; + border-bottom: 1px solid #e0e0e0; + } + + .cost-table th { + background: #f8f9fa; + font-weight: 600; + color: #555; + } + + .cost-table tr:last-child td { + border-bottom: none; + font-weight: 600; + } + + .cost-table tbody tr:hover { + background: #f8f9fa; + } + .error { background: #fee; color: #c00; @@ -464,6 +525,7 @@ +
@@ -493,6 +555,29 @@

🔗 Most Relevant Sources

+ + +
+
+ 💰 Cost Breakdown +
+ + + + + + + + + + + + + +
#TypeInputOutputCost (USD)
+
+
+
@@ -531,6 +616,10 @@ // SSE connection let eventSource = null; let logEntries = []; + + // Clarification state + let isAwaitingClarification = false; + let clarificationData = null; function connectToSSE() { eventSource = new EventSource('/stream'); @@ -679,8 +768,109 @@ searchBtn.disabled = false; } } + + async function performSearchWithClarification(clarificationText) { + if (!clarificationData || !clarificationData.originalQuestion) { + showError('Clarification data not available'); + return; + } + + // Show loading + hideError(); + hideResults(); + showLoading(); + + try { + const response = await fetch('/search', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + question: clarificationData.originalQuestion + ' ' + clarificationText, + previousClarification: clarificationText, + originalQuestion: clarificationData.originalQuestion + }) + }); + + const data = await response.json(); + + if (!response.ok) { + throw new Error(data.error || 'Search failed'); + } + + displayResults(data); + } catch (err) { + showError(err.message); + } finally { + hideLoading(); + } + } function displayResults(data) { + // Check if clarification is needed + if (data.clarificationNeeded) { + isAwaitingClarification = true; + clarificationData = { + originalQuestion: data.originalQuestion + }; + + // Show clarification prompt in the answer area + answerEl.innerHTML = ` +
+

${data.fullAnswerHTMLSnippet}

+ + +
+ `; + + // Clear sources since we're waiting for clarification + sourcesEl.innerHTML = '
  • Waiting for clarification...
    • '; + + // Add event listener for clarification submission + setTimeout(() => { + const clarificationInput = document.getElementById('clarificationInput'); + const clarificationSubmitBtn = document.getElementById('clarificationSubmitBtn'); + + if (clarificationInput && clarificationSubmitBtn) { + clarificationInput.focus(); + + const handleClarification = () => { + const clarificationText = clarificationInput.value.trim(); + if (clarificationText) { + // Search again with the clarification + performSearchWithClarification(clarificationText); + } + }; + + clarificationSubmitBtn.addEventListener('click', handleClarification); + clarificationInput.addEventListener('keypress', (e) => { + if (e.key === 'Enter') { + handleClarification(); + } + }); + } + }, 100); + + showResults(); + return; + } + + // Reset clarification state for normal results + isAwaitingClarification = false; + clarificationData = null; + // Display answer as HTML answerEl.innerHTML = data.fullAnswerHTMLSnippet || '

    No answer generated

    '; @@ -703,6 +893,27 @@ sourcesEl.appendChild(li); } + // Display cost breakdown + const costTableBody = document.getElementById('costTableBody'); + costTableBody.innerHTML = ''; + if (data.costBreakdown && data.costBreakdown.length > 0) { + data.costBreakdown.forEach(item => { + const tr = document.createElement('tr'); + tr.innerHTML = ` + ${item.index} + ${item.type} + ${item.prompt_tokens || '-'} + ${item.completion_tokens || '-'} + ${item.cost} + `; + costTableBody.appendChild(tr); + }); + } else { + const tr = document.createElement('tr'); + tr.innerHTML = 'No cost data available'; + costTableBody.appendChild(tr); + } + showResults(); } diff --git a/searchCLI.js b/searchCLI.js index 058936f..7ec2fc9 100644 --- a/searchCLI.js +++ b/searchCLI.js @@ -19,9 +19,6 @@ function createSimpleBroadcast() { const timestamp = new Date().toLocaleTimeString(); const prefix = type === 'error' ? '❌' : type === 'warning' ? '⚠️' : type === 'success' ? '✅' : 'ℹ️'; console.log(`[${timestamp}] ${prefix} ${message}`); - if (data) { - console.log(` Data: ${JSON.stringify(data, null, 2)}`); - } }; } diff --git a/src/routes/search.js b/src/routes/search.js index daac250..e8cc703 100644 --- a/src/routes/search.js +++ b/src/routes/search.js @@ -5,7 +5,7 @@ export function createSearchRouter(searchService, broadcast) { const router = Router(); router.post('/', async (request, response) => { - const { question } = request.body; + const { question, previousClarification, originalQuestion } = request.body; if (!question) { response.status(400).json({ @@ -16,7 +16,7 @@ export function createSearchRouter(searchService, broadcast) { } try { - const result = await searchService.search(question); + const result = await searchService.search(question, previousClarification, originalQuestion); response.json(result); } catch (error) { if (error instanceof SearchServiceError) { diff --git a/src/services/openRouterLogger.js b/src/services/openRouterLogger.js index a31621c..1ac16b7 100644 --- a/src/services/openRouterLogger.js +++ b/src/services/openRouterLogger.js @@ -33,7 +33,7 @@ function formatTimestamp() { * @param {Object} response - The response from OpenRouter * @param {Error} [error] - Optional error if the call failed */ -export async function logOpenRouterCall(operation, params, response, error = null) { +export async function logOpenRouterCall(operation, text, params, response, error = null) { try { await ensureLogDirectory(); @@ -43,7 +43,7 @@ export async function logOpenRouterCall(operation, params, response, error = nul const logContent = `${timestamp} | ${operation}\n\nParams:\n${JSON.stringify(params, null, 2)}\n\nResponse:\n${JSON.stringify(response, null, 2)}${error ? `\n\nError: ${error.message}` : ''}\n`; - await fs.writeFile(logFilePath, logContent, 'utf8'); + await fs.writeFile(logFilePath, text + logContent, 'utf8'); } catch (logError) { console.error('Failed to log OpenRouter call:', logError); // Don't throw - logging should not break the main functionality diff --git a/src/services/openRouterService.js b/src/services/openRouterService.js index 35c90d8..d8b22a9 100644 --- a/src/services/openRouterService.js +++ b/src/services/openRouterService.js @@ -2,8 +2,13 @@ import { logOpenRouterCall } from './openRouterLogger.js'; function parseResponse(response) { if(!response?.usage?.cost) console.log(response); - - return { cost: response?.usage?.cost, data: JSON.parse(response.choices[0].message.content) }; + console.log('OpenRouter API call cost:', response?.usage); + return { + cost: response?.usage?.cost, + prompt_tokens: response?.usage?.prompt_tokens, + completion_tokens: response?.usage?.completion_tokens, + data: JSON.parse(response.choices[0].message.content) + }; } export async function summarizeSources({ openrouter, text, question }) { @@ -60,7 +65,95 @@ export async function summarizeSources({ openrouter, text, question }) { body: JSON.stringify(params), }); const response = await fetchResponse.json(); - await logOpenRouterCall('summarizeSources', params, response); + await logOpenRouterCall('summarizeSources', text, params, response); + return parseResponse(response); +} + +export async function rephraseQuestion({ question, previousClarification, originalQuestion }) { + + if(previousClarification) { + const prompt = ` + You are a search query expert. You are given a question and you return + a search query for a web search engine that would return the best results to answer the question. + Do NOT restrict the query using site: operator. + Also give a list of 2 supplementary search queries to deepen the search. + Today is the date of ${new Date().toLocaleDateString()}. + The user has provided this clarification "${previousClarification}" to the original question: ` + originalQuestion; + + const params = { + model: 'openai/gpt-5.4-mini', + messages: [ + { role: 'system', content: prompt }, + { role: 'user', content: question }, + ], + reasoning: { effort: 'none' }, + stream: false, + response_format: { + type: 'json_schema', json_schema: { + name: 'queries', strict: true, schema: { + required: [ 'query', 'supplementaryQueries'], type: 'object', additionalProperties: false, properties: { + query: { type: 'string' }, + supplementaryQueries: { type: 'array', items: { type: 'string' } }, + } } } } + }; + + const apiKey = process.env.OPENROUTER_API_KEY; + const fetchResponse = await fetch('https://openrouter.ai/api/v1/chat/completions', { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(params), + }); + const response = await fetchResponse.json(); + await logOpenRouterCall('rephraseQuestion', question,params, response); + return parseResponse(response); + } + + + const prompt = ` + You are a search query expert. You are given a question and you return + a search query for a web search engine that would return the best results to answer the question. + Do NOT restrict the query using site: operator. + Also give a list of 2 supplementary search queries to deepen the search. + Today is the date of ${new Date().toLocaleDateString()}. + The user cannot ask questions that a web search engine cannot answer. + Like "Who are you". + If the question is ambiguous or unsuited for a web search, you can ask for clarification. + ${previousClarification ? `The user has provided this clarification to the original question: "${previousClarification}". Use this clarification to refine the search query.` : ''} + `; + + const params = { + model: 'openai/gpt-5.4-mini', + messages: [ + { role: 'system', content: prompt }, + { role: 'user', content: question }, + ], + reasoning: { effort: 'none' }, + stream: false, + response_format: { + type: 'json_schema', json_schema: { + name: 'queries', strict: true, schema: { + required: ['needsClarification', 'clarification', 'query', 'supplementaryQueries'], type: 'object', additionalProperties: false, properties: { + needsClarification: { type: 'boolean', description: 'Indicates if the question is ambiguous and needs clarification' }, + clarification: { type: 'string', description: 'If needsClarification is true, this field contains the clarification question to ask the user. Otherwise, it is an empty string.' }, + query: { type: 'string' }, + supplementaryQueries: { type: 'array', items: { type: 'string' } }, + } } } } + }; + + const apiKey = process.env.OPENROUTER_API_KEY; + const fetchResponse = await fetch('https://openrouter.ai/api/v1/chat/completions', { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(params), + }); + const response = await fetchResponse.json(); + await logOpenRouterCall('rephraseQuestion', question,params, response); return parseResponse(response); } @@ -68,8 +161,8 @@ export async function summarizeFinalAnswer({ openrouter, text, question }) { const prompt = ` You are a search result analyst. Today is the date of ${new Date().toLocaleDateString()}. Based on the following search results for the query "${question}", - Summarize the search results to answer the original query. Use Emoji and HTML. Tags allowed: , , ,
      ,
    • , ,

      - Also provide the most relevant sources. + Summarize the search results to answer the original query. Use Emoji and HTML. Tags allowed: , , , 
      , 
        , 
      • , , 
         
         
        
+    Also provide the most relevant sources. Answer in the language of the question.
   `;
 
   const params = {
@@ -112,7 +205,7 @@ export async function summarizeFinalAnswer({ openrouter, text, question }) {
   });
   const response = await fetchResponse.json();
   
-  await logOpenRouterCall('summarizeFinalAnswer', params, response);
+  await logOpenRouterCall('summarizeFinalAnswer', text, params, response);
   
   return parseResponse(response);
 }
diff --git a/src/services/searchFormatter.js b/src/services/searchFormatter.js
index 39e1c9e..44e2df6 100644
--- a/src/services/searchFormatter.js
+++ b/src/services/searchFormatter.js
@@ -19,7 +19,7 @@ function wrapText(text, maxLineLength = 68) {
   return output;
 }
 
-export function formatSummarySources(sources) {
+export function formatSummarySources(sources,supplementaryResults=[]) {
   let output = `\n${'='.repeat(70)}\n`;
   output += ` ${'SUMMARY'.padStart(35).padEnd(69)}\n`;
   output += `${'='.repeat(70)}\n`;
@@ -38,6 +38,13 @@ export function formatSummarySources(sources) {
     }
   });
 
+  if (supplementaryResults.length > 0) {
+    supplementaryResults.forEach((result) => {
+      output += `\n${'-'.repeat(70)}\n`;
+      output += JSON.stringify(result, null, 2);
+    });
+  }
+
   output += `\n${'='.repeat(70)}\n`;
   output += `Total sources: ${sources.length}\n`;
   output += `${'='.repeat(70)}\n`;
diff --git a/src/services/searchService.js b/src/services/searchService.js
index 1662db3..6c2a420 100644
--- a/src/services/searchService.js
+++ b/src/services/searchService.js
@@ -1,7 +1,16 @@
 import { extractContent } from './extractContent.js';
 import { logExtraction } from './extractionLogger.js';
-import { summarizeFinalAnswer, summarizeSources } from './openRouterService.js';
+import { summarizeFinalAnswer, summarizeSources, rephraseQuestion } from './openRouterService.js';
 import { formatSummarySources } from './searchFormatter.js';
+import { get_encoding } from 'tiktoken';
+
+const encoding = get_encoding('cl100k_base');
+
+function countTokens(text) {
+  if (!text) return 0;
+  const tokens = encoding.encode(text);
+  return tokens.length;
+}
 
 const EXA_SEARCH_OPTIONS = {
   numResults: 10,
@@ -72,18 +81,114 @@ function enrichSourcesWithDetails(sources, detailedContents) {
   }
 }
 
+// Helper function to build cost breakdown
+function buildCostBreakdown(cost) {
+  // Print cost breakdown as a formatted table
+  console.log('\n=== Cost Breakdown ===');
+  console.table(
+    cost.map((item, index) => ({
+      '#': index + 1,
+      Type: item.type,
+      'Cost (USD)': `$${item.amount.toFixed(6)}`,
+    })),
+  );
+  const totalCost = cost.reduce((sum, item) => sum + item.amount, 0);
+  console.log(`Total: $${totalCost.toFixed(6)}\n`);
+
+  // Build cost breakdown table for API response
+  const costBreakdown = cost.map((item, index) => ({
+    index: index + 1,
+    type: item.type,
+    prompt_tokens: item.prompt_tokens,
+    completion_tokens: item.completion_tokens,
+    cost: `$${item.amount.toFixed(6)}`,
+  }));
+
+  // Add total to the breakdown
+  costBreakdown.push({
+    index: costBreakdown.length + 1,
+    type: 'Total',
+    cost: `$${totalCost.toFixed(6)}`,
+  });
+
+  return costBreakdown;
+}
+
 export function createSearchService({ exa, openrouter, broadcast }) {
   return {
-    async search(question) {
-      broadcast(`🔍 Search request: "${question}"`, 'info', { question });
+    async search(question, previousClarification, originalQuestion) {
+      let finalSummary;
+      const cost=[];
+      let rephrasedQuestion;
 
-      broadcast('Searching with Exa...', 'info');
-      const result = await exa.search(question, EXA_SEARCH_OPTIONS);
+      try {
+        const rephraseResult = await rephraseQuestion({question, previousClarification, originalQuestion});
+        rephrasedQuestion = rephraseResult.data;
+        cost.push({
+          type:'openrouter_rephrase', 
+          amount: rephraseResult.cost,
+          prompt_tokens: rephraseResult.prompt_tokens,
+          completion_tokens: rephraseResult.completion_tokens
+        });
+      } catch (error) {
+        throw new SearchServiceError('Failed to generate summary', 500, error);
+      }
+      if (rephrasedQuestion.needsClarification) {
+        finalSummary = {
+          fullAnswerHTMLSnippet: rephrasedQuestion.clarification,
+          clarificationNeeded: true,
+          originalQuestion: question,
+          mostRelevantSources: [],
+        };
+        // Attach cost breakdown to the final summary
+        finalSummary.costBreakdown = buildCostBreakdown(cost);
+
+        return finalSummary;
+       
+      }
+
+      // Limit to first 2 supplementary queries
+      const limitedQueries = rephrasedQuestion.supplementaryQueries.slice(0, 2);
+      //limitedQueries.push(rephrasedQuestion.query);
+      const supplementaryResults = [];
+      for (const item of limitedQueries) {
+        console.log('Supplementary Query:', item);
+        const result = await exa.search(item, {
+          numResults: 2,
+          type: 'auto',
+          contents: {
+            highlights: {
+              maxCharacters: 2000,
+            },
+          },
+        });
+        supplementaryResults.push({
+          query: item,
+          highlights: result.results.map(r => r.highlights),
+        });
+        const outputTokens = countTokens(JSON.stringify(result.results));
+        cost.push({ 
+          type: 'exa_search_complement', 
+          amount: result.costDollars.total,
+          prompt_tokens: 0,
+          completion_tokens: outputTokens
+        });
+      }
+
+
+      broadcast('Searching with Exa for '+rephrasedQuestion.query, 'info');
+      const result = await exa.search(rephrasedQuestion.query, EXA_SEARCH_OPTIONS);
       broadcast(`✅ Found ${result.results.length} results`, 'success', {
         count: result.results.length,
       });
-      const cost=[];
-      cost.push({type:'exa_search', amount: result.costDollars.total});
+      
+      const outputTokens = countTokens(JSON.stringify(result.results));
+      cost.push({
+        type:'exa_search', 
+        amount: result.costDollars.total,
+        prompt_tokens: 0,
+        completion_tokens: outputTokens
+      });
 
       const extractedContent = extractContent(result);
 
@@ -105,13 +210,18 @@ export function createSearchService({ exa, openrouter, broadcast }) {
           question,
         });
         summary = summaryResult.data;
-        cost.push({type:'openrouter_summarize', amount: summaryResult.cost});
+        cost.push({
+          type:'openrouter_rank', 
+          amount: summaryResult.cost,
+          prompt_tokens: summaryResult.prompt_tokens,
+          completion_tokens: summaryResult.completion_tokens
+        });
       } catch (error) {
         throw new SearchServiceError('Failed to generate summary', 500, error);
       }
 
       if (!summary?.sources) {
-        throw new SearchServiceError('Failed to generate summary');
+        throw new SearchServiceError('Failed to generate summary for query: ' + question + '. No sources returned:'+ JSON.stringify(summary, null, 2));
       }
 
       broadcast(`✅ Generated summaries for ${summary.sources.length} sources`, 'success', {
@@ -126,25 +236,35 @@ export function createSearchService({ exa, openrouter, broadcast }) {
         broadcast,
       });
 
-      broadcast('🔧 Enhancing summaries with detailed content...', 'info');
       enrichSourcesWithDetails(summary.sources, detailedContents);
       for (const detailed of detailedContents) {
         if (detailed.cost) {
-          cost.push({type:'exa_get_content', amount: detailed.cost});
+          const outputTokens = countTokens(JSON.stringify(detailed.content));
+          cost.push({
+            type:'exa_get_content', 
+            amount: detailed.cost,
+            prompt_tokens: 0,
+            completion_tokens: outputTokens
+          });
         }
       }
 
       broadcast('📊 Generating final summary...', 'info');
 
-      let finalSummary;
+      
       try {
         const finalSummaryResult = await summarizeFinalAnswer({
           openrouter,
-          text: formatSummarySources(summary.sources),
+          text: formatSummarySources(summary.sources,supplementaryResults),
           question,
         });
         finalSummary = finalSummaryResult.data;
-        cost.push({type:'openrouter_final_summary', amount: finalSummaryResult.cost});
+        cost.push({
+          type:'openrouter_final_summary', 
+          amount: finalSummaryResult.cost,
+          prompt_tokens: finalSummaryResult.prompt_tokens,
+          completion_tokens: finalSummaryResult.completion_tokens
+        });
       } catch (error) {
         throw new SearchServiceError('Failed to generate final summary', 500, error);
       }
@@ -158,6 +278,9 @@ export function createSearchService({ exa, openrouter, broadcast }) {
         sourcesCount: finalSummary.mostRelevantSources?.length || 0,
       });
 
+      // Attach cost breakdown to the final summary
+      finalSummary.costBreakdown = buildCostBreakdown(cost);
+
       return finalSummary;
     },
   };
diff --git a/src/utils/htmlConsoleRenderer.js b/src/utils/htmlConsoleRenderer.js
index 7ab9cc5..a0c48f8 100644
--- a/src/utils/htmlConsoleRenderer.js
+++ b/src/utils/htmlConsoleRenderer.js
@@ -59,6 +59,14 @@ export function renderHTML(html) {
   // Handle 
        content
         - block with newline
   text = text.replace(/
        ([\s\S]*?)<\/div>/gi, (match, content) => `${content.trim()}\n\n`);
 
+  // Handle 
        content
         - preformatted text with monospace styling
+  text = text.replace(/
        ([\s\S]*?)<\/pre>/gi, (match, content) => {
+    // Preserve internal whitespace and newlines, apply dimmed styling for code-like appearance
+    const lines = content.split('\n');
+    const formattedLines = lines.map(line => chalk.dim(line));
+    return '\n' + formattedLines.join('\n') + '\n\n';
+  });
+
   // Clean up any remaining HTML tags that weren't processed
   text = text.replace(/<[^>]+>/g, '');