token count for search

2026-04-04 21:51:52 +02:00
parent 92b313fa79
commit d2920fd39a
10 changed files with 477 additions and 30 deletions
--- a/package-lock.json
+++ b/package-lock.json
@@ -13,7 +13,8 @@
        "chalk": "^5.6.2",
        "dotenv": "^17.2.3",
        "exa-js": "^2.11.0",
-        "express": "^5.2.1"
+        "express": "^5.2.1",
+        "tiktoken": "^1.0.22"
      }
    },
    "node_modules/@openrouter/sdk": {
@@ -889,6 +890,12 @@
        "node": ">= 0.8"
      }
    },
+    "node_modules/tiktoken": {
+      "version": "1.0.22",
+      "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.22.tgz",
+      "integrity": "sha512-PKvy1rVF1RibfF3JlXBSP0Jrcw2uq3yXdgcEXtKTYn3QJ/cBRBHDnrJ5jHky+MENZ6DIPwNUGWpkVx+7joCpNA==",
+      "license": "MIT"
+    },
    "node_modules/toidentifier": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",