unify parse api
This commit is contained in:
seb
@@ -80,7 +80,7 @@ Generated from `@openapi` annotations in [`src/api/routes/`](../../src/api/route
|
|||||||
| [`routes/customergroup/update.md`](routes/customergroup/update.md) | `PATCH /customergroup/{id}` |
|
| [`routes/customergroup/update.md`](routes/customergroup/update.md) | `PATCH /customergroup/{id}` |
|
||||||
| [`routes/delivery/list.md`](routes/delivery/list.md) | `GET /delivery` |
|
| [`routes/delivery/list.md`](routes/delivery/list.md) | `GET /delivery` |
|
||||||
| [`routes/delivery/create.md`](routes/delivery/create.md) | `POST /delivery` |
|
| [`routes/delivery/create.md`](routes/delivery/create.md) | `POST /delivery` |
|
||||||
| [`routes/delivery/parsePDF.md`](routes/delivery/parsePDF.md) | `POST /delivery/parsePDF` |
|
| [`routes/delivery/parse.md`](routes/delivery/parse.md) | `POST /delivery/parse` |
|
||||||
| [`routes/delivery/delete.md`](routes/delivery/delete.md) | `DELETE /delivery/{id}` |
|
| [`routes/delivery/delete.md`](routes/delivery/delete.md) | `DELETE /delivery/{id}` |
|
||||||
| [`routes/delivery/get.md`](routes/delivery/get.md) | `GET /delivery/{id}` |
|
| [`routes/delivery/get.md`](routes/delivery/get.md) | `GET /delivery/{id}` |
|
||||||
| [`routes/delivery/update.md`](routes/delivery/update.md) | `PATCH /delivery/{id}` |
|
| [`routes/delivery/update.md`](routes/delivery/update.md) | `PATCH /delivery/{id}` |
|
||||||
|
|||||||
@@ -4258,23 +4258,23 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"/delivery/parsePDF": {
|
"/delivery/parse": {
|
||||||
"post": {
|
"post": {
|
||||||
"tags": [
|
"tags": [
|
||||||
"Delivery"
|
"Delivery"
|
||||||
],
|
],
|
||||||
"summary": "Parse PDF (SSE)",
|
"summary": "Parse document (SSE)",
|
||||||
"description": "Accepts a PDF invoice or delivery note (`multipart/form-data`, field `pdf`)\nand streams extraction results as Server-Sent Events until parsing is complete.\n\nThe PDF is read via `pdftotext -layout` and fed page-by-page to an LLM\n(OpenRouter) which fills a structured manifest of header data and line items.\nThe OpenRouter API key is read from the authenticated user's company setting\n`keys.openrouter`; the endpoint returns `400` if the key is not configured.\n\nThe stream ends after the final `result` event (or an `error` event on failure).\n",
|
"description": "Accepts an invoice or delivery note as PDF or text file\n(`multipart/form-data`, field `file`) and streams extraction results as\nServer-Sent Events until parsing is complete.\n\nPDFs are read via `pdftotext -layout`; text files are decoded as UTF-8.\nPages are separated by the form feed character (`\\f`). If no form feed\nis present, the input is treated as a single page.\n\nThe extracted pages are fed to an LLM (OpenRouter) which fills a structured\nmanifest of header data and line items. The OpenRouter API key is read from\nthe authenticated user's company setting `keys.openrouter`; the endpoint\nreturns `400` if the key is not configured.\n\nThe stream ends after the final `result` event (or an `error` event on failure).\n",
|
||||||
"security": [
|
"security": [
|
||||||
{
|
{
|
||||||
"cookieAuth": []
|
"cookieAuth": []
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"x-markdown": "delivery/parsePDF.md",
|
"x-markdown": "delivery/parse.md",
|
||||||
"x-sse-events": [
|
"x-sse-events": [
|
||||||
{
|
{
|
||||||
"name": "status",
|
"name": "status",
|
||||||
"description": "PDF parse confirmation and LLM status messages",
|
"description": "Document parse confirmation and LLM status messages",
|
||||||
"payload": "{ \"message\": \"…\" }"
|
"payload": "{ \"message\": \"…\" }"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -4304,7 +4304,7 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "error",
|
"name": "error",
|
||||||
"description": "Emitted on PDF parse or LLM failure; stream closes immediately after",
|
"description": "Emitted on document parse or LLM failure; stream closes immediately after",
|
||||||
"payload": "{ \"message\": \"…\" }"
|
"payload": "{ \"message\": \"…\" }"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
@@ -4315,13 +4315,13 @@
|
|||||||
"schema": {
|
"schema": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
"pdf"
|
"file"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"pdf": {
|
"file": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"format": "binary",
|
"format": "binary",
|
||||||
"description": "PDF file to parse (max 50 MB, application/pdf)"
|
"description": "Document to parse (max 50 MB, application/pdf or text file)"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -4340,7 +4340,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"400": {
|
"400": {
|
||||||
"description": "No PDF uploaded or OpenRouter key not configured",
|
"description": "No file uploaded, unsupported file type, or OpenRouter key not configured",
|
||||||
"content": {
|
"content": {
|
||||||
"application/json": {
|
"application/json": {
|
||||||
"schema": {
|
"schema": {
|
||||||
|
|||||||
@@ -1,16 +1,21 @@
|
|||||||
# `POST /delivery/parsePDF`
|
# `POST /delivery/parse`
|
||||||
|
|
||||||
Protected. Requires a valid session cookie (`credentials: 'include'`).
|
Protected. Requires a valid session cookie (`credentials: 'include'`).
|
||||||
|
|
||||||
Parse PDF (SSE)
|
Parse document (SSE)
|
||||||
|
|
||||||
Accepts a PDF invoice or delivery note (`multipart/form-data`, field `pdf`)
|
Accepts an invoice or delivery note as PDF or text file
|
||||||
and streams extraction results as Server-Sent Events until parsing is complete.
|
(`multipart/form-data`, field `file`) and streams extraction results as
|
||||||
|
Server-Sent Events until parsing is complete.
|
||||||
|
|
||||||
The PDF is read via `pdftotext -layout` and fed page-by-page to an LLM
|
PDFs are read via `pdftotext -layout`; text files are decoded as UTF-8.
|
||||||
(OpenRouter) which fills a structured manifest of header data and line items.
|
Pages are separated by the form feed character (`\f`). If no form feed
|
||||||
The OpenRouter API key is read from the authenticated user's company setting
|
is present, the input is treated as a single page.
|
||||||
`keys.openrouter`; the endpoint returns `400` if the key is not configured.
|
|
||||||
|
The extracted pages are fed to an LLM (OpenRouter) which fills a structured
|
||||||
|
manifest of header data and line items. The OpenRouter API key is read from
|
||||||
|
the authenticated user's company setting `keys.openrouter`; the endpoint
|
||||||
|
returns `400` if the key is not configured.
|
||||||
|
|
||||||
The stream ends after the final `result` event (or an `error` event on failure).
|
The stream ends after the final `result` event (or an `error` event on failure).
|
||||||
|
|
||||||
@@ -19,7 +24,7 @@ The stream ends after the final `result` event (or an `error` event on failure).
|
|||||||
|
|
||||||
| Field | Required | Type | Description |
|
| Field | Required | Type | Description |
|
||||||
|-------|----------|------|-------------|
|
|-------|----------|------|-------------|
|
||||||
| `pdf` | yes | file | PDF file to parse (max 50 MB, application/pdf) |
|
| `file` | yes | file | Document to parse (max 50 MB, application/pdf or text file) |
|
||||||
|
|
||||||
## SSE Events
|
## SSE Events
|
||||||
|
|
||||||
@@ -27,13 +32,13 @@ The response is a `text/event-stream`. Events are emitted in order until the str
|
|||||||
|
|
||||||
| Event | Description | Payload |
|
| Event | Description | Payload |
|
||||||
|-------|-------------|---------|
|
|-------|-------------|---------|
|
||||||
| `status` | PDF parse confirmation and LLM status messages | `{ "message": "…" }` |
|
| `status` | Document parse confirmation and LLM status messages | `{ "message": "…" }` |
|
||||||
| `llm_call` | Fired before each LLM request | `{ "model": "…", "promptSize": 0, "messages": 0 }` |
|
| `llm_call` | Fired before each LLM request | `{ "model": "…", "promptSize": 0, "messages": 0 }` |
|
||||||
| `tool_call` | Fired when the output_manifest tool is invoked | `{ "name": "output_manifest" }` |
|
| `tool_call` | Fired when the output_manifest tool is invoked | `{ "name": "output_manifest" }` |
|
||||||
| `usage` | Token usage after each LLM response | `{ "input": 0, "cached": 0, "output": 0, "cost": 0 }` |
|
| `usage` | Token usage after each LLM response | `{ "input": 0, "cached": 0, "output": 0, "cost": 0 }` |
|
||||||
| `progress` | Intermediate accumulated extraction state (basedata / notes / items) | `ManifestResult` |
|
| `progress` | Intermediate accumulated extraction state (basedata / notes / items) | `ManifestResult` |
|
||||||
| `result` | Final accumulated extraction — emitted just before the stream closes | `ManifestResult` |
|
| `result` | Final accumulated extraction — emitted just before the stream closes | `ManifestResult` |
|
||||||
| `error` | Emitted on PDF parse or LLM failure; stream closes immediately after | `{ "message": "…" }` |
|
| `error` | Emitted on document parse or LLM failure; stream closes immediately after | `{ "message": "…" }` |
|
||||||
|
|
||||||
## Response `200`
|
## Response `200`
|
||||||
|
|
||||||
@@ -77,7 +82,7 @@ The response is a `text/event-stream`. Events are emitted in order until the str
|
|||||||
|
|
||||||
| Status | Description |
|
| Status | Description |
|
||||||
|--------|-------------|
|
|--------|-------------|
|
||||||
| 400 | No PDF uploaded or OpenRouter key not configured — `{"error":"…"}` |
|
| 400 | No file uploaded, unsupported file type, or OpenRouter key not configured — `{"error":"…"}` |
|
||||||
| 401 | Unauthorized — `{"error":"…"}` |
|
| 401 | Unauthorized — `{"error":"…"}` |
|
||||||
|
|
||||||
---
|
---
|
||||||
463
src/api/routes/deliveryParse.js
Normal file
463
src/api/routes/deliveryParse.js
Normal file
@@ -0,0 +1,463 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
|
import { execFileSync } from 'child_process';
|
||||||
|
import multer from 'multer';
|
||||||
|
import { LLMOrbiter, OpenRouterProvider, defineTool, DirectReturnError } from 'libllmorbiter';
|
||||||
|
import * as dbCompanySetting from '../../db/companysetting.js';
|
||||||
|
|
||||||
|
// ─── Multer — memory storage (no temp files); accepts PDF or text ───────────────
|
||||||
|
const upload = multer({
|
||||||
|
storage: multer.memoryStorage(),
|
||||||
|
limits: { fileSize: 50 * 1024 * 1024 }, // 50 MB
|
||||||
|
fileFilter: (_req, file, cb) => {
|
||||||
|
const name = file.originalname?.toLowerCase() || '';
|
||||||
|
const isPdf = file.mimetype === 'application/pdf' || name.endsWith('.pdf');
|
||||||
|
const isText =
|
||||||
|
file.mimetype.startsWith('text/') ||
|
||||||
|
/\.(txt|text|md|csv|log|json)$/.test(name);
|
||||||
|
if (isPdf || isText) {
|
||||||
|
cb(null, true);
|
||||||
|
} else {
|
||||||
|
cb(new Error('Only PDF or text files are accepted'));
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
/** Detects whether an uploaded file is a PDF (else treated as text). */
|
||||||
|
function isPdfUpload(file) {
|
||||||
|
const name = file.originalname?.toLowerCase() || '';
|
||||||
|
return file.mimetype === 'application/pdf' || name.endsWith('.pdf');
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── SSE helpers ────────────────────────────────────────────────────────────────
|
||||||
|
function sseWrite(res, event, data) {
|
||||||
|
if (res.writableEnded) return;
|
||||||
|
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── PDF → pages via pdftotext ──────────────────────────────────────────────────
|
||||||
|
function readPdfPages(buffer) {
|
||||||
|
// Write buffer to stdin by passing '-' as output and reading from stdin via
|
||||||
|
// execFileSync input option. pdftotext reads from '-' for stdin.
|
||||||
|
const raw = execFileSync('pdftotext', ['-layout', '-', '-'], {
|
||||||
|
input: buffer,
|
||||||
|
encoding: 'utf8',
|
||||||
|
maxBuffer: 50 * 1024 * 1024,
|
||||||
|
});
|
||||||
|
return raw
|
||||||
|
.split('\f')
|
||||||
|
.map((p) => p.trim())
|
||||||
|
.filter((p) => p.length > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Text → pages split by form feed (same convention as pdftotext) ─────────────
|
||||||
|
function readTxtPages(textOrBuffer) {
|
||||||
|
const text = Buffer.isBuffer(textOrBuffer) ? textOrBuffer.toString('utf8') : String(textOrBuffer);
|
||||||
|
return text
|
||||||
|
.split('\f')
|
||||||
|
.map((p) => p.trim())
|
||||||
|
.filter((p) => p.length > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Extraction prompt ──────────────────────────────────────────────────────────
|
||||||
|
const BASE_PROMPT = `Extract metadata and product line items from the provided invoice or delivery note.
|
||||||
|
Call the output_manifest tool with the extracted data. Do not answer with plain text once extraction is complete.
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
- manifestType: "rechnung" or "lieferschein". For "lieferschein", prices and taxes must be null.
|
||||||
|
- Exclude non-product lines (shipping, handling, etc.).
|
||||||
|
- Line items: Extract count, vpe (default: 1), liefArtNr, mwst (7 or 19), and eknetto per piece(euro cents).
|
||||||
|
- Match supplier against the provided list for kLieferant and lieferantName. For unknown suppliers, extract lieferantName and set kLieferant to null.
|
||||||
|
- Write notes strictly in German.
|
||||||
|
- The output_manifest tool has three strictly optional components: basedata, notes, items. Only include a component when it carries genuinely new data for this batch; omit the rest.
|
||||||
|
- Include basedata ONLY when at least one of its fields is currently null/empty and you can fill it in, or when a value actually changes. If all basedata fields are already set and unchanged, omit basedata.
|
||||||
|
- Include items ONLY when this batch contains new product line items not already present. If there are no new items, omit items.
|
||||||
|
- Include notes ONLY when there is a genuinely new noteworthy detail not already in the notes list. These notes are for a human to double-check the extraction result, so record anything ambiguous, unusual, or worth verifying (e.g. packaging interpretations, excluded lines, unavailable articles).
|
||||||
|
|
||||||
|
VPE (Verpackungseinheit) Logic:
|
||||||
|
VPE represents the number of individually sellable units inside a bulk item. If not explicitly split for individual sale, default to 1.
|
||||||
|
- "20 Boxen á 10 Stück" -> VPE = 20.
|
||||||
|
- "10 Gläser á 100 Filter" -> VPE = 10 (we sell the 10 glasses individually).
|
||||||
|
- "100 Stück" -> VPE = 1 (sold as one bulk pack).
|
||||||
|
- We don't sell the individual filters or Pieces, but typical VPEs like Beutel, Glässer, Boxen, Hefte
|
||||||
|
- That means that in "Y Hefte á X Blatt" the VPE is NOT X but: Y
|
||||||
|
- a so called "Display" also means VPE. So "20er Display" means VPE = 20
|
||||||
|
- eknetto is the price of one full vpe
|
||||||
|
|
||||||
|
Unit (ME) rules:
|
||||||
|
- The unit (ME) must ALWAYS refer to the individual sellable item (e.g. Stk., Beutel, Flasche, Box, Set). It must NEVER be "VE" or "VPE".
|
||||||
|
- A packaging size like "VE100" means the bulk pack contains 100 individual items: set vpe = 100 and unit = the individual item (e.g. Stk.).
|
||||||
|
- Example: "Einhängesiebe Stahl, VE100" -> vpe = 100, unit = "Stk.", count = 1 (one bulk pack of 100 sieves).`;
|
||||||
|
|
||||||
|
// ─── Shared SSE manifest extraction (POST /delivery/parse) ─────────────────────
|
||||||
|
/**
|
||||||
|
* Fetches the OpenRouter API key, opens the SSE stream, and runs the LLM
|
||||||
|
* manifest extraction over the given pages. Closes the stream when done.
|
||||||
|
*
|
||||||
|
* @param {object} opts
|
||||||
|
* @param {import('express').Request} opts.req
|
||||||
|
* @param {import('express').Response} opts.res
|
||||||
|
* @param {string[]} opts.pages Trimmed, non-empty document pages
|
||||||
|
* @param {string} opts.sourceLabel e.g. 'PDF' or 'Text'
|
||||||
|
*/
|
||||||
|
async function parseDocumentManifest({ req, res, pages, sourceLabel }) {
|
||||||
|
// ── Fetch OpenRouter API key from company settings ────────────────
|
||||||
|
let apiKey;
|
||||||
|
try {
|
||||||
|
const setting = await dbCompanySetting.getByKey(req.user.id, 'keys.openrouter', { maskWriteonly: false });
|
||||||
|
apiKey = setting?.value;
|
||||||
|
} catch (err) {
|
||||||
|
console.error('delivery parse: failed to read OpenRouter key:', err);
|
||||||
|
}
|
||||||
|
if (!apiKey) {
|
||||||
|
return res.status(400).json({ error: 'OpenRouter API key not configured (companysetting: keys.openrouter)' });
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Open SSE stream ──────────────────────────────────────────────
|
||||||
|
res.setHeader('Content-Type', 'text/event-stream');
|
||||||
|
res.setHeader('Cache-Control', 'no-cache');
|
||||||
|
res.setHeader('Connection', 'keep-alive');
|
||||||
|
res.flushHeaders();
|
||||||
|
|
||||||
|
sseWrite(res, 'status', { message: `${sourceLabel} parsed: ${pages.length} page(s)` });
|
||||||
|
|
||||||
|
// ── Accumulated state ────────────────────────────────────────────
|
||||||
|
const manifestBasedata = {
|
||||||
|
date: null,
|
||||||
|
kLieferant: null,
|
||||||
|
lieferantName: null,
|
||||||
|
manifestNr: null,
|
||||||
|
manifestType: null,
|
||||||
|
};
|
||||||
|
const manifestNotes = [];
|
||||||
|
const manifestItems = { items: [], pagesFrom: null, pagesTo: null };
|
||||||
|
let currentPage = 0;
|
||||||
|
|
||||||
|
// ── Tool definition ──────────────────────────────────────────────
|
||||||
|
const outputManifestTool = defineTool({
|
||||||
|
name: 'output_manifest',
|
||||||
|
description:
|
||||||
|
'Stores extracted invoice/delivery-note data: header metadata (basedata), notes, and product line items. Each component is strictly optional — only include the ones that carry new data.',
|
||||||
|
strict: false,
|
||||||
|
parameters: {
|
||||||
|
type: 'object',
|
||||||
|
additionalProperties: false,
|
||||||
|
required: [],
|
||||||
|
properties: {
|
||||||
|
basedata: {
|
||||||
|
anyOf: [
|
||||||
|
{
|
||||||
|
type: 'object',
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: {
|
||||||
|
date: { type: ['string', 'null'], description: 'Document date (ISO 8601)' },
|
||||||
|
kLieferant: { type: ['number', 'null'], description: 'Lieferant ID' },
|
||||||
|
lieferantName: { type: 'string', description: 'Lieferant name' },
|
||||||
|
manifestNr: { type: ['string', 'null'], description: 'Lieferscheinnummer or Rechnungsnummer' },
|
||||||
|
manifestType: {
|
||||||
|
anyOf: [
|
||||||
|
{ type: 'string', enum: ['rechnung', 'lieferschein'] },
|
||||||
|
{ type: 'null' },
|
||||||
|
],
|
||||||
|
description: 'Document type',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{ type: 'null' },
|
||||||
|
],
|
||||||
|
description: 'Header/metadata. Only include fields that are currently missing or changed.',
|
||||||
|
},
|
||||||
|
notes: {
|
||||||
|
anyOf: [
|
||||||
|
{ type: 'array', items: { type: 'string' } },
|
||||||
|
{ type: 'null' },
|
||||||
|
],
|
||||||
|
description: 'New noteworthy details (in German), for a human to double-check the result.',
|
||||||
|
},
|
||||||
|
items: {
|
||||||
|
anyOf: [
|
||||||
|
{
|
||||||
|
type: 'object',
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: {
|
||||||
|
numberOfItemsToAdd: { type: 'number', description: 'Number of NEW line items being added in this call.' },
|
||||||
|
items: {
|
||||||
|
type: 'array',
|
||||||
|
description: 'Invoice/delivery product line items',
|
||||||
|
items: {
|
||||||
|
type: 'object',
|
||||||
|
additionalProperties: false,
|
||||||
|
required: ['position', 'count', 'vpe', 'unit', 'articleName', 'liefArtNr', 'mwst', 'eknetto'],
|
||||||
|
properties: {
|
||||||
|
position: { type: 'number', description: 'Position' },
|
||||||
|
count: { type: 'number', description: 'Quantity (Menge)' },
|
||||||
|
vpe: { type: 'number', description: 'Packaging unit size (VPE). Default: 1.' },
|
||||||
|
unit: { type: 'string', description: 'Unit of the individual sellable item. NEVER "VE" or "VPE".' },
|
||||||
|
articleName: { type: 'string', description: 'Article name' },
|
||||||
|
liefArtNr: { type: 'string', description: 'Supplier article number' },
|
||||||
|
mwst: {
|
||||||
|
anyOf: [{ type: 'string', enum: ['7', '19'] }, { type: 'null' }],
|
||||||
|
description: 'VAT rate; null if missing',
|
||||||
|
},
|
||||||
|
eknetto: { type: ['number', 'null'], description: 'Net price per vpe in euro cents; null if missing' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{ type: 'null' },
|
||||||
|
],
|
||||||
|
description: 'New product line items.',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
invoke: async (args) => {
|
||||||
|
// Merge basedata
|
||||||
|
if (args.basedata && typeof args.basedata === 'object') {
|
||||||
|
for (const key of ['date', 'kLieferant', 'lieferantName', 'manifestNr', 'manifestType']) {
|
||||||
|
if (args.basedata[key] != null) manifestBasedata[key] = args.basedata[key];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Merge notes
|
||||||
|
if (Array.isArray(args.notes)) {
|
||||||
|
for (const n of args.notes) {
|
||||||
|
if (n && !manifestNotes.includes(n)) manifestNotes.push(n);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Merge items
|
||||||
|
if (args.items && typeof args.items === 'object' && Array.isArray(args.items.items)) {
|
||||||
|
const existing = new Set(manifestItems.items.map((it) => it.position));
|
||||||
|
for (const it of args.items.items) {
|
||||||
|
if (!existing.has(it.position)) {
|
||||||
|
manifestItems.items.push({ ...it, page: currentPage });
|
||||||
|
existing.add(it.position);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const itemPages = manifestItems.items.map((it) => it.page).filter((p) => p != null);
|
||||||
|
if (itemPages.length) {
|
||||||
|
manifestItems.pagesFrom = Math.min(...itemPages);
|
||||||
|
manifestItems.pagesTo = Math.max(...itemPages);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Emit intermediate progress
|
||||||
|
sseWrite(res, 'progress', {
|
||||||
|
basedata: manifestBasedata,
|
||||||
|
notes: manifestNotes,
|
||||||
|
items: manifestItems,
|
||||||
|
});
|
||||||
|
|
||||||
|
throw new DirectReturnError({
|
||||||
|
basedata: manifestBasedata,
|
||||||
|
notes: manifestNotes,
|
||||||
|
items: manifestItems,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── LLMOrbiter setup ─────────────────────────────────────────────
|
||||||
|
const model = '~openai/gpt-luna-latest:floor';
|
||||||
|
const provider = new OpenRouterProvider({ apiKey });
|
||||||
|
const orbiter = new LLMOrbiter(provider, {
|
||||||
|
onStatus: (status) => sseWrite(res, 'status', { message: status }),
|
||||||
|
onLLMCall: (call) =>
|
||||||
|
sseWrite(res, 'llm_call', {
|
||||||
|
model: call.model,
|
||||||
|
promptSize: call.promptSize,
|
||||||
|
messages: call.messages,
|
||||||
|
}),
|
||||||
|
onToolCall: (call) => sseWrite(res, 'tool_call', { name: call.name }),
|
||||||
|
onUsage: (usage) =>
|
||||||
|
sseWrite(res, 'usage', {
|
||||||
|
input: usage.input,
|
||||||
|
cached: usage.cached,
|
||||||
|
output: usage.output,
|
||||||
|
cost: usage.cost,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Process all pages in one batch ───────────────────────────────
|
||||||
|
try {
|
||||||
|
const pageContent = pages
|
||||||
|
.map((p, idx) => `--- Page ${idx + 1} of ${pages.length} ---\n${p}`)
|
||||||
|
.join('\n\n');
|
||||||
|
|
||||||
|
const prompt = `${BASE_PROMPT}
|
||||||
|
|
||||||
|
You are processing all ${pages.length} page(s) of this document.
|
||||||
|
|
||||||
|
Current manifest_basedata (fill in what is still missing):
|
||||||
|
${JSON.stringify(manifestBasedata, null, 2)}
|
||||||
|
|
||||||
|
Current notes:
|
||||||
|
${JSON.stringify(manifestNotes, null, 2)}
|
||||||
|
|
||||||
|
Current manifest_items:
|
||||||
|
${JSON.stringify(manifestItems, null, 2)}
|
||||||
|
|
||||||
|
Page content:
|
||||||
|
${pageContent}`;
|
||||||
|
|
||||||
|
currentPage = 1;
|
||||||
|
await orbiter.run({ prompt, model, tools: [outputManifestTool] });
|
||||||
|
} catch (err) {
|
||||||
|
if (!res.writableEnded) {
|
||||||
|
sseWrite(res, 'error', { message: 'LLM processing failed: ' + err.message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Final result ─────────────────────────────────────────────────
|
||||||
|
sseWrite(res, 'result', {
|
||||||
|
basedata: manifestBasedata,
|
||||||
|
notes: manifestNotes,
|
||||||
|
items: manifestItems,
|
||||||
|
});
|
||||||
|
|
||||||
|
res.end();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Route ───────────────────────────────────────────────────────────────────────
|
||||||
|
/**
|
||||||
|
* @param {import('express').Application} app
|
||||||
|
* @param {string} [basePath='']
|
||||||
|
*/
|
||||||
|
export function register(app, basePath = '') {
|
||||||
|
/**
|
||||||
|
* @openapi
|
||||||
|
* /delivery/parse:
|
||||||
|
* post:
|
||||||
|
* tags: [Delivery]
|
||||||
|
* summary: Parse document (SSE)
|
||||||
|
* description: |
|
||||||
|
* Accepts an invoice or delivery note as PDF or text file
|
||||||
|
* (`multipart/form-data`, field `file`) and streams extraction results as
|
||||||
|
* Server-Sent Events until parsing is complete.
|
||||||
|
*
|
||||||
|
* PDFs are read via `pdftotext -layout`; text files are decoded as UTF-8.
|
||||||
|
* Pages are separated by the form feed character (`\f`). If no form feed
|
||||||
|
* is present, the input is treated as a single page.
|
||||||
|
*
|
||||||
|
* The extracted pages are fed to an LLM (OpenRouter) which fills a structured
|
||||||
|
* manifest of header data and line items. The OpenRouter API key is read from
|
||||||
|
* the authenticated user's company setting `keys.openrouter`; the endpoint
|
||||||
|
* returns `400` if the key is not configured.
|
||||||
|
*
|
||||||
|
* The stream ends after the final `result` event (or an `error` event on failure).
|
||||||
|
* security:
|
||||||
|
* - cookieAuth: []
|
||||||
|
* x-markdown: delivery/parse.md
|
||||||
|
* x-sse-events:
|
||||||
|
* - name: status
|
||||||
|
* description: Document parse confirmation and LLM status messages
|
||||||
|
* payload: '{ "message": "…" }'
|
||||||
|
* - name: llm_call
|
||||||
|
* description: Fired before each LLM request
|
||||||
|
* payload: '{ "model": "…", "promptSize": 0, "messages": 0 }'
|
||||||
|
* - name: tool_call
|
||||||
|
* description: Fired when the output_manifest tool is invoked
|
||||||
|
* payload: '{ "name": "output_manifest" }'
|
||||||
|
* - name: usage
|
||||||
|
* description: Token usage after each LLM response
|
||||||
|
* payload: '{ "input": 0, "cached": 0, "output": 0, "cost": 0 }'
|
||||||
|
* - name: progress
|
||||||
|
* description: Intermediate accumulated extraction state (basedata / notes / items)
|
||||||
|
* payload: ManifestResult
|
||||||
|
* - name: result
|
||||||
|
* description: Final accumulated extraction — emitted just before the stream closes
|
||||||
|
* payload: ManifestResult
|
||||||
|
* - name: error
|
||||||
|
* description: Emitted on document parse or LLM failure; stream closes immediately after
|
||||||
|
* payload: '{ "message": "…" }'
|
||||||
|
* requestBody:
|
||||||
|
* required: true
|
||||||
|
* content:
|
||||||
|
* multipart/form-data:
|
||||||
|
* schema:
|
||||||
|
* type: object
|
||||||
|
* required: [file]
|
||||||
|
* properties:
|
||||||
|
* file:
|
||||||
|
* type: string
|
||||||
|
* format: binary
|
||||||
|
* description: Document to parse (max 50 MB, application/pdf or text file)
|
||||||
|
* responses:
|
||||||
|
* 200:
|
||||||
|
* description: |
|
||||||
|
* `text/event-stream` — emits `status`, `llm_call`, `tool_call`, `usage`,
|
||||||
|
* `progress`, `result` (and `error` on failure). See SSE Events table below.
|
||||||
|
*
|
||||||
|
* **ManifestResult shape** (used by `progress` and `result`):
|
||||||
|
*
|
||||||
|
* ```json
|
||||||
|
* {
|
||||||
|
* "basedata": {
|
||||||
|
* "date": "2026-01-15",
|
||||||
|
* "kLieferant": 42,
|
||||||
|
* "lieferantName": "Muster GmbH",
|
||||||
|
* "manifestNr": "LS-2026-001",
|
||||||
|
* "manifestType": "lieferschein"
|
||||||
|
* },
|
||||||
|
* "notes": ["Pos. 3: VPE-Angabe fehlte, Standardwert 1 angenommen."],
|
||||||
|
* "items": {
|
||||||
|
* "pagesFrom": 1,
|
||||||
|
* "pagesTo": 2,
|
||||||
|
* "items": [
|
||||||
|
* {
|
||||||
|
* "position": 1,
|
||||||
|
* "count": 10,
|
||||||
|
* "vpe": 1,
|
||||||
|
* "unit": "Stk.",
|
||||||
|
* "articleName": "Einhängesieb Stahl",
|
||||||
|
* "liefArtNr": "ES-100",
|
||||||
|
* "mwst": "19",
|
||||||
|
* "eknetto": 450,
|
||||||
|
* "page": 1
|
||||||
|
* }
|
||||||
|
* ]
|
||||||
|
* }
|
||||||
|
* }
|
||||||
|
* ```
|
||||||
|
* content:
|
||||||
|
* text/event-stream:
|
||||||
|
* schema:
|
||||||
|
* type: string
|
||||||
|
* 400:
|
||||||
|
* description: No file uploaded, unsupported file type, or OpenRouter key not configured
|
||||||
|
* content:
|
||||||
|
* application/json:
|
||||||
|
* schema:
|
||||||
|
* $ref: '#/components/schemas/Error'
|
||||||
|
* 401:
|
||||||
|
* description: Unauthorized
|
||||||
|
* content:
|
||||||
|
* application/json:
|
||||||
|
* schema:
|
||||||
|
* $ref: '#/components/schemas/Error'
|
||||||
|
*/
|
||||||
|
app.post(
|
||||||
|
`${basePath}/delivery/parse`,
|
||||||
|
upload.single('file'),
|
||||||
|
async (req, res) => {
|
||||||
|
if (!req.file?.buffer) {
|
||||||
|
return res.status(400).json({ error: 'A document file is required (field name: file)' });
|
||||||
|
}
|
||||||
|
|
||||||
|
let pages;
|
||||||
|
try {
|
||||||
|
pages = isPdfUpload(req.file) ? readPdfPages(req.file.buffer) : readTxtPages(req.file.buffer);
|
||||||
|
} catch (err) {
|
||||||
|
return res.status(400).json({ error: 'Failed to parse document: ' + err.message });
|
||||||
|
}
|
||||||
|
if (pages.length === 0) {
|
||||||
|
return res.status(400).json({ error: 'Document content is empty' });
|
||||||
|
}
|
||||||
|
|
||||||
|
await parseDocumentManifest({
|
||||||
|
req,
|
||||||
|
res,
|
||||||
|
pages,
|
||||||
|
sourceLabel: isPdfUpload(req.file) ? 'PDF' : 'Text',
|
||||||
|
});
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -1,420 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
import { execFileSync } from 'child_process';
|
|
||||||
import multer from 'multer';
|
|
||||||
import { LLMOrbiter, OpenRouterProvider, defineTool, DirectReturnError } from 'libllmorbiter';
|
|
||||||
import * as dbCompanySetting from '../../db/companysetting.js';
|
|
||||||
|
|
||||||
// ─── Multer — memory storage (no temp files) ────────────────────────────────────
|
|
||||||
const upload = multer({
|
|
||||||
storage: multer.memoryStorage(),
|
|
||||||
limits: { fileSize: 50 * 1024 * 1024 }, // 50 MB
|
|
||||||
fileFilter: (_req, file, cb) => {
|
|
||||||
if (file.mimetype === 'application/pdf' || file.originalname?.toLowerCase().endsWith('.pdf')) {
|
|
||||||
cb(null, true);
|
|
||||||
} else {
|
|
||||||
cb(new Error('Only PDF files are accepted'));
|
|
||||||
}
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
// ─── SSE helpers ────────────────────────────────────────────────────────────────
|
|
||||||
function sseWrite(res, event, data) {
|
|
||||||
if (res.writableEnded) return;
|
|
||||||
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── PDF → pages via pdftotext ──────────────────────────────────────────────────
|
|
||||||
function readPdfPages(buffer) {
|
|
||||||
// Write buffer to stdin by passing '-' as output and reading from stdin via
|
|
||||||
// execFileSync input option. pdftotext reads from '-' for stdin.
|
|
||||||
const raw = execFileSync('pdftotext', ['-layout', '-', '-'], {
|
|
||||||
input: buffer,
|
|
||||||
encoding: 'utf8',
|
|
||||||
maxBuffer: 50 * 1024 * 1024,
|
|
||||||
});
|
|
||||||
return raw
|
|
||||||
.split('\f')
|
|
||||||
.map((p) => p.trim())
|
|
||||||
.filter((p) => p.length > 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Extraction prompt ──────────────────────────────────────────────────────────
|
|
||||||
const BASE_PROMPT = `Extract metadata and product line items from the provided invoice or delivery note.
|
|
||||||
Call the output_manifest tool with the extracted data. Do not answer with plain text once extraction is complete.
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
- manifestType: "rechnung" or "lieferschein". For "lieferschein", prices and taxes must be null.
|
|
||||||
- Exclude non-product lines (shipping, handling, etc.).
|
|
||||||
- Line items: Extract count, vpe (default: 1), liefArtNr, mwst (7 or 19), and eknetto per piece(euro cents).
|
|
||||||
- Match supplier against the provided list for kLieferant and lieferantName. For unknown suppliers, extract lieferantName and set kLieferant to null.
|
|
||||||
- Write notes strictly in German.
|
|
||||||
- The output_manifest tool has three strictly optional components: basedata, notes, items. Only include a component when it carries genuinely new data for this batch; omit the rest.
|
|
||||||
- Include basedata ONLY when at least one of its fields is currently null/empty and you can fill it in, or when a value actually changes. If all basedata fields are already set and unchanged, omit basedata.
|
|
||||||
- Include items ONLY when this batch contains new product line items not already present. If there are no new items, omit items.
|
|
||||||
- Include notes ONLY when there is a genuinely new noteworthy detail not already in the notes list. These notes are for a human to double-check the extraction result, so record anything ambiguous, unusual, or worth verifying (e.g. packaging interpretations, excluded lines, unavailable articles).
|
|
||||||
|
|
||||||
VPE (Verpackungseinheit) Logic:
|
|
||||||
VPE represents the number of individually sellable units inside a bulk item. If not explicitly split for individual sale, default to 1.
|
|
||||||
- "20 Boxen á 10 Stück" -> VPE = 20.
|
|
||||||
- "10 Gläser á 100 Filter" -> VPE = 10 (we sell the 10 glasses individually).
|
|
||||||
- "100 Stück" -> VPE = 1 (sold as one bulk pack).
|
|
||||||
- We don't sell the individual filters or Pieces, but typical VPEs like Beutel, Glässer, Boxen, Hefte
|
|
||||||
- That means that in "Y Hefte á X Blatt" the VPE is NOT X but: Y
|
|
||||||
- a so called "Display" also means VPE. So "20er Display" means VPE = 20
|
|
||||||
- eknetto is the price of one full vpe
|
|
||||||
|
|
||||||
Unit (ME) rules:
|
|
||||||
- The unit (ME) must ALWAYS refer to the individual sellable item (e.g. Stk., Beutel, Flasche, Box, Set). It must NEVER be "VE" or "VPE".
|
|
||||||
- A packaging size like "VE100" means the bulk pack contains 100 individual items: set vpe = 100 and unit = the individual item (e.g. Stk.).
|
|
||||||
- Example: "Einhängesiebe Stahl, VE100" -> vpe = 100, unit = "Stk.", count = 1 (one bulk pack of 100 sieves).`;
|
|
||||||
|
|
||||||
// ─── Route ───────────────────────────────────────────────────────────────────────
|
|
||||||
/**
|
|
||||||
* @param {import('express').Application} app
|
|
||||||
* @param {string} [basePath='']
|
|
||||||
*/
|
|
||||||
export function register(app, basePath = '') {
|
|
||||||
/**
|
|
||||||
* @openapi
|
|
||||||
* /delivery/parsePDF:
|
|
||||||
* post:
|
|
||||||
* tags: [Delivery]
|
|
||||||
* summary: Parse PDF (SSE)
|
|
||||||
* description: |
|
|
||||||
* Accepts a PDF invoice or delivery note (`multipart/form-data`, field `pdf`)
|
|
||||||
* and streams extraction results as Server-Sent Events until parsing is complete.
|
|
||||||
*
|
|
||||||
* The PDF is read via `pdftotext -layout` and fed page-by-page to an LLM
|
|
||||||
* (OpenRouter) which fills a structured manifest of header data and line items.
|
|
||||||
* The OpenRouter API key is read from the authenticated user's company setting
|
|
||||||
* `keys.openrouter`; the endpoint returns `400` if the key is not configured.
|
|
||||||
*
|
|
||||||
* The stream ends after the final `result` event (or an `error` event on failure).
|
|
||||||
* security:
|
|
||||||
* - cookieAuth: []
|
|
||||||
* x-markdown: delivery/parsePDF.md
|
|
||||||
* x-sse-events:
|
|
||||||
* - name: status
|
|
||||||
* description: PDF parse confirmation and LLM status messages
|
|
||||||
* payload: '{ "message": "…" }'
|
|
||||||
* - name: llm_call
|
|
||||||
* description: Fired before each LLM request
|
|
||||||
* payload: '{ "model": "…", "promptSize": 0, "messages": 0 }'
|
|
||||||
* - name: tool_call
|
|
||||||
* description: Fired when the output_manifest tool is invoked
|
|
||||||
* payload: '{ "name": "output_manifest" }'
|
|
||||||
* - name: usage
|
|
||||||
* description: Token usage after each LLM response
|
|
||||||
* payload: '{ "input": 0, "cached": 0, "output": 0, "cost": 0 }'
|
|
||||||
* - name: progress
|
|
||||||
* description: Intermediate accumulated extraction state (basedata / notes / items)
|
|
||||||
* payload: ManifestResult
|
|
||||||
* - name: result
|
|
||||||
* description: Final accumulated extraction — emitted just before the stream closes
|
|
||||||
* payload: ManifestResult
|
|
||||||
* - name: error
|
|
||||||
* description: Emitted on PDF parse or LLM failure; stream closes immediately after
|
|
||||||
* payload: '{ "message": "…" }'
|
|
||||||
* requestBody:
|
|
||||||
* required: true
|
|
||||||
* content:
|
|
||||||
* multipart/form-data:
|
|
||||||
* schema:
|
|
||||||
* type: object
|
|
||||||
* required: [pdf]
|
|
||||||
* properties:
|
|
||||||
* pdf:
|
|
||||||
* type: string
|
|
||||||
* format: binary
|
|
||||||
* description: PDF file to parse (max 50 MB, application/pdf)
|
|
||||||
* responses:
|
|
||||||
* 200:
|
|
||||||
* description: |
|
|
||||||
* `text/event-stream` — emits `status`, `llm_call`, `tool_call`, `usage`,
|
|
||||||
* `progress`, `result` (and `error` on failure). See SSE Events table below.
|
|
||||||
*
|
|
||||||
* **ManifestResult shape** (used by `progress` and `result`):
|
|
||||||
*
|
|
||||||
* ```json
|
|
||||||
* {
|
|
||||||
* "basedata": {
|
|
||||||
* "date": "2026-01-15",
|
|
||||||
* "kLieferant": 42,
|
|
||||||
* "lieferantName": "Muster GmbH",
|
|
||||||
* "manifestNr": "LS-2026-001",
|
|
||||||
* "manifestType": "lieferschein"
|
|
||||||
* },
|
|
||||||
* "notes": ["Pos. 3: VPE-Angabe fehlte, Standardwert 1 angenommen."],
|
|
||||||
* "items": {
|
|
||||||
* "pagesFrom": 1,
|
|
||||||
* "pagesTo": 2,
|
|
||||||
* "items": [
|
|
||||||
* {
|
|
||||||
* "position": 1,
|
|
||||||
* "count": 10,
|
|
||||||
* "vpe": 1,
|
|
||||||
* "unit": "Stk.",
|
|
||||||
* "articleName": "Einhängesieb Stahl",
|
|
||||||
* "liefArtNr": "ES-100",
|
|
||||||
* "mwst": "19",
|
|
||||||
* "eknetto": 450,
|
|
||||||
* "page": 1
|
|
||||||
* }
|
|
||||||
* ]
|
|
||||||
* }
|
|
||||||
* }
|
|
||||||
* ```
|
|
||||||
* content:
|
|
||||||
* text/event-stream:
|
|
||||||
* schema:
|
|
||||||
* type: string
|
|
||||||
* 400:
|
|
||||||
* description: No PDF uploaded or OpenRouter key not configured
|
|
||||||
* content:
|
|
||||||
* application/json:
|
|
||||||
* schema:
|
|
||||||
* $ref: '#/components/schemas/Error'
|
|
||||||
* 401:
|
|
||||||
* description: Unauthorized
|
|
||||||
* content:
|
|
||||||
* application/json:
|
|
||||||
* schema:
|
|
||||||
* $ref: '#/components/schemas/Error'
|
|
||||||
*/
|
|
||||||
app.post(
|
|
||||||
`${basePath}/delivery/parsePDF`,
|
|
||||||
upload.single('pdf'),
|
|
||||||
async (req, res) => {
|
|
||||||
// ── Validate upload ──────────────────────────────────────────────
|
|
||||||
if (!req.file) {
|
|
||||||
return res.status(400).json({ error: 'A PDF file is required (field name: pdf)' });
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Fetch OpenRouter API key from company settings ────────────────
|
|
||||||
let apiKey;
|
|
||||||
try {
|
|
||||||
const setting = await dbCompanySetting.getByKey(req.user.id, 'keys.openrouter', { maskWriteonly: false });
|
|
||||||
apiKey = setting?.value;
|
|
||||||
console.log('parsePDF debug: user=', req.user.id, 'setting=', setting ? { ...setting, value: setting.value ? `${setting.value.slice(0, 8)}... (len=${setting.value.length})` : null } : null);
|
|
||||||
} catch (err) {
|
|
||||||
console.error('parsePDF: failed to read OpenRouter key:', err);
|
|
||||||
}
|
|
||||||
if (!apiKey) {
|
|
||||||
return res.status(400).json({ error: 'OpenRouter API key not configured (companysetting: keys.openrouter)' });
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Open SSE stream ──────────────────────────────────────────────
|
|
||||||
res.setHeader('Content-Type', 'text/event-stream');
|
|
||||||
res.setHeader('Cache-Control', 'no-cache');
|
|
||||||
res.setHeader('Connection', 'keep-alive');
|
|
||||||
res.flushHeaders();
|
|
||||||
|
|
||||||
// ── Parse PDF ────────────────────────────────────────────────────
|
|
||||||
let pages;
|
|
||||||
try {
|
|
||||||
pages = readPdfPages(req.file.buffer);
|
|
||||||
} catch (err) {
|
|
||||||
sseWrite(res, 'error', { message: 'Failed to parse PDF: ' + err.message });
|
|
||||||
res.end();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
sseWrite(res, 'status', { message: `PDF parsed: ${pages.length} page(s)` });
|
|
||||||
|
|
||||||
// ── Accumulated state ────────────────────────────────────────────
|
|
||||||
const manifestBasedata = {
|
|
||||||
date: null,
|
|
||||||
kLieferant: null,
|
|
||||||
lieferantName: null,
|
|
||||||
manifestNr: null,
|
|
||||||
manifestType: null,
|
|
||||||
};
|
|
||||||
const manifestNotes = [];
|
|
||||||
const manifestItems = { items: [], pagesFrom: null, pagesTo: null };
|
|
||||||
let currentPage = 0;
|
|
||||||
|
|
||||||
// ── Tool definition ──────────────────────────────────────────────
|
|
||||||
const outputManifestTool = defineTool({
|
|
||||||
name: 'output_manifest',
|
|
||||||
description:
|
|
||||||
'Stores extracted invoice/delivery-note data: header metadata (basedata), notes, and product line items. Each component is strictly optional — only include the ones that carry new data.',
|
|
||||||
strict: false,
|
|
||||||
parameters: {
|
|
||||||
type: 'object',
|
|
||||||
additionalProperties: false,
|
|
||||||
required: [],
|
|
||||||
properties: {
|
|
||||||
basedata: {
|
|
||||||
anyOf: [
|
|
||||||
{
|
|
||||||
type: 'object',
|
|
||||||
additionalProperties: false,
|
|
||||||
properties: {
|
|
||||||
date: { type: ['string', 'null'], description: 'Document date (ISO 8601)' },
|
|
||||||
kLieferant: { type: ['number', 'null'], description: 'Lieferant ID' },
|
|
||||||
lieferantName: { type: 'string', description: 'Lieferant name' },
|
|
||||||
manifestNr: { type: ['string', 'null'], description: 'Lieferscheinnummer or Rechnungsnummer' },
|
|
||||||
manifestType: {
|
|
||||||
anyOf: [
|
|
||||||
{ type: 'string', enum: ['rechnung', 'lieferschein'] },
|
|
||||||
{ type: 'null' },
|
|
||||||
],
|
|
||||||
description: 'Document type',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{ type: 'null' },
|
|
||||||
],
|
|
||||||
description: 'Header/metadata. Only include fields that are currently missing or changed.',
|
|
||||||
},
|
|
||||||
notes: {
|
|
||||||
anyOf: [
|
|
||||||
{ type: 'array', items: { type: 'string' } },
|
|
||||||
{ type: 'null' },
|
|
||||||
],
|
|
||||||
description: 'New noteworthy details (in German), for a human to double-check the result.',
|
|
||||||
},
|
|
||||||
items: {
|
|
||||||
anyOf: [
|
|
||||||
{
|
|
||||||
type: 'object',
|
|
||||||
additionalProperties: false,
|
|
||||||
properties: {
|
|
||||||
numberOfItemsToAdd: { type: 'number', description: 'Number of NEW line items being added in this call.' },
|
|
||||||
items: {
|
|
||||||
type: 'array',
|
|
||||||
description: 'Invoice/delivery product line items',
|
|
||||||
items: {
|
|
||||||
type: 'object',
|
|
||||||
additionalProperties: false,
|
|
||||||
required: ['position', 'count', 'vpe', 'unit', 'articleName', 'liefArtNr', 'mwst', 'eknetto'],
|
|
||||||
properties: {
|
|
||||||
position: { type: 'number', description: 'Position' },
|
|
||||||
count: { type: 'number', description: 'Quantity (Menge)' },
|
|
||||||
vpe: { type: 'number', description: 'Packaging unit size (VPE). Default: 1.' },
|
|
||||||
unit: { type: 'string', description: 'Unit of the individual sellable item. NEVER "VE" or "VPE".' },
|
|
||||||
articleName: { type: 'string', description: 'Article name' },
|
|
||||||
liefArtNr: { type: 'string', description: 'Supplier article number' },
|
|
||||||
mwst: {
|
|
||||||
anyOf: [{ type: 'string', enum: ['7', '19'] }, { type: 'null' }],
|
|
||||||
description: 'VAT rate; null if missing',
|
|
||||||
},
|
|
||||||
eknetto: { type: ['number', 'null'], description: 'Net price per vpe in euro cents; null if missing' },
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{ type: 'null' },
|
|
||||||
],
|
|
||||||
description: 'New product line items.',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
invoke: async (args) => {
|
|
||||||
// Merge basedata
|
|
||||||
if (args.basedata && typeof args.basedata === 'object') {
|
|
||||||
for (const key of ['date', 'kLieferant', 'lieferantName', 'manifestNr', 'manifestType']) {
|
|
||||||
if (args.basedata[key] != null) manifestBasedata[key] = args.basedata[key];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Merge notes
|
|
||||||
if (Array.isArray(args.notes)) {
|
|
||||||
for (const n of args.notes) {
|
|
||||||
if (n && !manifestNotes.includes(n)) manifestNotes.push(n);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Merge items
|
|
||||||
if (args.items && typeof args.items === 'object' && Array.isArray(args.items.items)) {
|
|
||||||
const existing = new Set(manifestItems.items.map((it) => it.position));
|
|
||||||
for (const it of args.items.items) {
|
|
||||||
if (!existing.has(it.position)) {
|
|
||||||
manifestItems.items.push({ ...it, page: currentPage });
|
|
||||||
existing.add(it.position);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const pages = manifestItems.items.map((it) => it.page).filter((p) => p != null);
|
|
||||||
if (pages.length) {
|
|
||||||
manifestItems.pagesFrom = Math.min(...pages);
|
|
||||||
manifestItems.pagesTo = Math.max(...pages);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Emit intermediate progress
|
|
||||||
sseWrite(res, 'progress', {
|
|
||||||
basedata: manifestBasedata,
|
|
||||||
notes: manifestNotes,
|
|
||||||
items: manifestItems,
|
|
||||||
});
|
|
||||||
|
|
||||||
throw new DirectReturnError({
|
|
||||||
basedata: manifestBasedata,
|
|
||||||
notes: manifestNotes,
|
|
||||||
items: manifestItems,
|
|
||||||
});
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
// ── LLMOrbiter setup ─────────────────────────────────────────────
|
|
||||||
const model = '~openai/gpt-luna-latest:floor';
|
|
||||||
const provider = new OpenRouterProvider({ apiKey });
|
|
||||||
const orbiter = new LLMOrbiter(provider, {
|
|
||||||
onStatus: (status) => sseWrite(res, 'status', { message: status }),
|
|
||||||
onLLMCall: (call) =>
|
|
||||||
sseWrite(res, 'llm_call', {
|
|
||||||
model: call.model,
|
|
||||||
promptSize: call.promptSize,
|
|
||||||
messages: call.messages,
|
|
||||||
}),
|
|
||||||
onToolCall: (call) => sseWrite(res, 'tool_call', { name: call.name }),
|
|
||||||
onUsage: (usage) =>
|
|
||||||
sseWrite(res, 'usage', {
|
|
||||||
input: usage.input,
|
|
||||||
cached: usage.cached,
|
|
||||||
output: usage.output,
|
|
||||||
cost: usage.cost,
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
// ── Process all pages in one batch ───────────────────────────────
|
|
||||||
try {
|
|
||||||
const pageContent = pages
|
|
||||||
.map((p, idx) => `--- Page ${idx + 1} of ${pages.length} ---\n${p}`)
|
|
||||||
.join('\n\n');
|
|
||||||
|
|
||||||
const prompt = `${BASE_PROMPT}
|
|
||||||
|
|
||||||
You are processing all ${pages.length} page(s) of this document.
|
|
||||||
|
|
||||||
Current manifest_basedata (fill in what is still missing):
|
|
||||||
${JSON.stringify(manifestBasedata, null, 2)}
|
|
||||||
|
|
||||||
Current notes:
|
|
||||||
${JSON.stringify(manifestNotes, null, 2)}
|
|
||||||
|
|
||||||
Current manifest_items:
|
|
||||||
${JSON.stringify(manifestItems, null, 2)}
|
|
||||||
|
|
||||||
Page content:
|
|
||||||
${pageContent}`;
|
|
||||||
|
|
||||||
currentPage = 1;
|
|
||||||
await orbiter.run({ prompt, model, tools: [outputManifestTool] });
|
|
||||||
} catch (err) {
|
|
||||||
if (!res.writableEnded) {
|
|
||||||
sseWrite(res, 'error', { message: 'LLM processing failed: ' + err.message });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Final result ─────────────────────────────────────────────────
|
|
||||||
sseWrite(res, 'result', {
|
|
||||||
basedata: manifestBasedata,
|
|
||||||
notes: manifestNotes,
|
|
||||||
items: manifestItems,
|
|
||||||
});
|
|
||||||
|
|
||||||
res.end();
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user