From b69110f1c2343fd6d8b384ef7049ed42bb45834c Mon Sep 17 00:00:00 2001 From: seb Date: Fri, 18 Sep 2026 08:28:44 +0200 Subject: [PATCH] fllabcks hmmm --- src/store/delivery/attachments.js | 22 ++- src/store/manifestattachment/add.js | 22 ++- src/util/generateDocumentThumb.js | 270 ++++++++++++++++++++++++---- 3 files changed, 276 insertions(+), 38 deletions(-) diff --git a/src/store/delivery/attachments.js b/src/store/delivery/attachments.js index cd638a2..43a1afe 100644 --- a/src/store/delivery/attachments.js +++ b/src/store/delivery/attachments.js @@ -60,9 +60,23 @@ function normalizeAttachment(raw, index) { if (raw.content == null) { throw new Error('Invalid file'); } - const content = Buffer.isBuffer(raw.content) - ? raw.content - : Buffer.from(raw.content); + let content; + if (Buffer.isBuffer(raw.content)) { + content = raw.content; + } else if (typeof raw.content === 'string') { + const trimmed = raw.content.trim(); + if (mimetype === 'application/pdf' || trimmed.startsWith('JVBER') || /^[A-Za-z0-9+/=]+$/.test(trimmed.slice(0, 100))) { + try { + content = Buffer.from(trimmed, 'base64'); + } catch { + content = Buffer.from(raw.content, 'utf8'); + } + } else { + content = Buffer.from(raw.content, 'utf8'); + } + } else { + content = Buffer.from(raw.content); + } const size = Number.isFinite(raw.size) && raw.size >= 0 ? Number(raw.size) @@ -137,7 +151,7 @@ export async function insertAttachments(pool, tx, sql, manifestId, files) { nextSort = Math.max(nextSort, sort) + 1; // Generate thumbnail server-side (e.g. pdftocairo + sharp for PDF) - const { thumbavif, thumbx, thumby } = await generateDocumentThumb(a.content, a.mimetype); + const { thumbavif, thumbx, thumby } = await generateDocumentThumb(a.content, a.mimetype, { filename: a.name }); const fileRes = await request(pool, tx) .input('hash', a.hash) diff --git a/src/store/manifestattachment/add.js b/src/store/manifestattachment/add.js index 1cc9e46..c4b63b1 100644 --- a/src/store/manifestattachment/add.js +++ b/src/store/manifestattachment/add.js @@ -77,9 +77,23 @@ export default function add(pool) { if (rawContent == null) { throw new Error('Content required'); } - const content = Buffer.isBuffer(rawContent) - ? rawContent - : Buffer.from(rawContent); + let content; + if (Buffer.isBuffer(rawContent)) { + content = rawContent; + } else if (typeof rawContent === 'string') { + const trimmed = rawContent.trim(); + if (mimetype === 'application/pdf' || trimmed.startsWith('JVBER') || /^[A-Za-z0-9+/=]+$/.test(trimmed.slice(0, 100))) { + try { + content = Buffer.from(trimmed, 'base64'); + } catch { + content = Buffer.from(rawContent, 'utf8'); + } + } else { + content = Buffer.from(rawContent, 'utf8'); + } + } else { + content = Buffer.from(rawContent); + } if (rawSort !== undefined && rawSort !== null) { if (!Number.isFinite(rawSort)) { @@ -143,7 +157,7 @@ export default function add(pool) { } if (!thumbavif) { - const generated = await generateDocumentThumb(content, mimetype); + const generated = await generateDocumentThumb(content, mimetype, { filename: rawName }); thumbavif = generated.thumbavif; thumbx = generated.thumbx; thumby = generated.thumby; diff --git a/src/util/generateDocumentThumb.js b/src/util/generateDocumentThumb.js index f3ac7ee..81f3a22 100644 --- a/src/util/generateDocumentThumb.js +++ b/src/util/generateDocumentThumb.js @@ -1,15 +1,45 @@ import sharp from 'sharp'; import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; const THUMB_MAX = 200; +export const DEBUG_DIR = process.env.PDF_DEBUG_DIR || path.resolve('debug/pdf'); + +/** + * Sanitizes a filename for filesystem storage. + * @param {string} [name] + * @returns {string} + */ +export function sanitizeFilename(name) { + if (!name || typeof name !== 'string') return 'document'; + return path.basename(name).replace(/[^a-zA-Z0-9._-]/g, '_'); +} + +/** + * Safely writes a buffer or string to disk, ensuring directory exists. + * @param {string} filePath + * @param {Buffer|string} data + */ +export function writeDebugFile(filePath, data) { + try { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, data); + return true; + } catch (err) { + console.warn(`[pdfDebug] Failed to write debug file ${filePath}:`, err.message); + return false; + } +} /** * Renders page 1 of a PDF into a PNG buffer using pdftocairo. * @param {Buffer} pdfBuffer - * @returns {Promise} + * @returns {Promise<{pngBuffer: Buffer, stderr: string, durationMs: number}>} */ -function renderPdfPageToPng(pdfBuffer) { +export function renderPdfPageToPng(pdfBuffer) { return new Promise((resolve, reject) => { + const startTime = Date.now(); const proc = spawn('pdftocairo', ['-png', '-singlefile', '-', '-'], { stdio: ['pipe', 'pipe', 'pipe'], }); @@ -20,66 +50,246 @@ function renderPdfPageToPng(pdfBuffer) { proc.stdout.on('data', (chunk) => chunks.push(chunk)); proc.stderr.on('data', (chunk) => errChunks.push(chunk)); + proc.stdin.on('error', (err) => { + // Prevent unhandled EPIPE crashes if pdftocairo exits early + console.warn('[pdfDebug][pdftocairo] stdin stream error:', err.message); + }); + proc.on('error', (err) => { reject(new Error(`Failed to start pdftocairo: ${err.message}`)); }); proc.on('close', (code) => { + const durationMs = Date.now() - startTime; + const stderr = Buffer.concat(errChunks).toString('utf8').trim(); if (code !== 0) { - const stderr = Buffer.concat(errChunks).toString('utf8').trim(); reject(new Error(`pdftocairo exited with code ${code}${stderr ? `: ${stderr}` : ''}`)); } else { - resolve(Buffer.concat(chunks)); + const pngBuffer = Buffer.concat(chunks); + resolve({ pngBuffer, stderr, durationMs }); } }); - proc.stdin.end(pdfBuffer); + try { + proc.stdin.end(pdfBuffer); + } catch (writeErr) { + console.warn('[pdfDebug][pdftocairo] Failed writing to stdin:', writeErr.message); + } }); } /** - * Generates an AVIF thumbnail for a document (PDF or image). - * Returns { thumbavif, thumbx, thumby } or nulls if unsupported or rendering fails. - * - * @param {Buffer} content - * @param {string} mimetype - * @returns {Promise<{thumbavif: Buffer|null, thumbx: number|null, thumby: number|null}>} + * Decodes content to a Buffer. If content is a base64 string, converts it to Buffer. + * @param {Buffer|string} rawContent + * @param {string} [mimetype] + * @returns {Buffer} */ -export default async function generateDocumentThumb(content, mimetype) { - if (!Buffer.isBuffer(content) || content.length === 0) { - return { thumbavif: null, thumbx: null, thumby: null }; +export function normalizeContentBuffer(rawContent, mimetype) { + if (Buffer.isBuffer(rawContent)) { + return rawContent; } - const type = typeof mimetype === 'string' ? mimetype.trim().toLowerCase() : ''; - - let imageBuffer = null; - if (type === 'application/pdf') { - try { - imageBuffer = await renderPdfPageToPng(content); - } catch (err) { - console.warn('generateDocumentThumb: PDF render failed:', err.message); - return { thumbavif: null, thumbx: null, thumby: null }; + if (typeof rawContent === 'string') { + const trimmed = rawContent.trim(); + // If it starts with %PDF or text header: + if (trimmed.startsWith('%PDF-')) { + return Buffer.from(rawContent, 'utf8'); } - } else if (type.startsWith('image/')) { - imageBuffer = content; - } else { - // Other formats (e.g. text/plain) do not have an image thumbnail - return { thumbavif: null, thumbx: null, thumby: null }; + // If PDF or looks like base64, attempt base64 decode + const isPdf = mimetype === 'application/pdf' || trimmed.startsWith('JVBER'); + if (isPdf || /^[A-Za-z0-9+/=]+$/.test(trimmed.slice(0, 100))) { + try { + const b64 = Buffer.from(trimmed, 'base64'); + if (b64.length > 0) { + return b64; + } + } catch { + // Fall back to utf8 + } + } + return Buffer.from(rawContent, 'utf8'); } + return Buffer.from(rawContent || ''); +} + +/** + * Debugs PDF upload and thumbnail generation: + * 1. Writes source PDF to disk + * 2. Renders page 1 PNG via pdftocairo and writes to disk + * 3. Converts to AVIF thumbnail via sharp and writes to disk + * 4. Writes complete JSON metadata to disk + * + * @param {Buffer|string} rawContent + * @param {object} [options] + * @param {string} [options.filename='upload.pdf'] + * @param {string} [options.mimetype='application/pdf'] + * @param {string} [options.debugDir] + * @param {boolean} [options.saveToDisk=true] + * @returns {Promise<{thumbavif: Buffer|null, thumbx: number|null, thumby: number|null, meta: object}>} + */ +export async function debugGeneratePdfThumbnail(rawContent, options = {}) { + const startTime = Date.now(); + const filename = options.filename || 'upload.pdf'; + const mimetype = options.mimetype || 'application/pdf'; + const saveToDisk = options.saveToDisk ?? true; + const targetDir = options.debugDir || DEBUG_DIR; + + const buffer = normalizeContentBuffer(rawContent, mimetype); + const magic = buffer.slice(0, 8).toString('utf8').replace(/[^\x20-\x7E]/g, '.'); + const isRealPdf = buffer.slice(0, 4).toString('utf8') === '%PDF'; + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const safeBase = `${timestamp}_${sanitizeFilename(filename)}`; + const debugBasePath = path.join(targetDir, safeBase); + + console.log(`[pdfDebug] ========================================================`); + console.log(`[pdfDebug] Processing PDF thumbnail for "${filename}"`); + console.log(`[pdfDebug] Mimetype: ${mimetype}, Size: ${buffer.length} bytes, Magic: "${magic}", Valid PDF Header: ${isRealPdf}`); + + const metaLog = { + timestamp: new Date().toISOString(), + filename, + mimetype, + sizeBytes: buffer.length, + magic, + isRealPdf, + savedFiles: {}, + }; + + // 1. Write source PDF to disk + if (saveToDisk) { + const sourcePdfPath = `${debugBasePath}-source.pdf`; + writeDebugFile(sourcePdfPath, buffer); + metaLog.savedFiles.sourcePdf = sourcePdfPath; + console.log(`[pdfDebug] Saved source PDF: ${sourcePdfPath}`); + } + + // 2. Render page 1 PNG via pdftocairo + let imageBuffer = null; try { + const pdfRenderInfo = await renderPdfPageToPng(buffer); + imageBuffer = pdfRenderInfo.pngBuffer; + + metaLog.pdftocairo = { + pngBytes: imageBuffer.length, + durationMs: pdfRenderInfo.durationMs, + stderr: pdfRenderInfo.stderr || null, + }; + + console.log(`[pdfDebug] pdftocairo rendered page 1 in ${pdfRenderInfo.durationMs}ms (${imageBuffer.length} bytes PNG)`); + if (pdfRenderInfo.stderr) { + console.warn(`[pdfDebug] pdftocairo stderr: ${pdfRenderInfo.stderr}`); + } + + if (saveToDisk) { + const page1PngPath = `${debugBasePath}-page1.png`; + writeDebugFile(page1PngPath, imageBuffer); + metaLog.savedFiles.page1Png = page1PngPath; + console.log(`[pdfDebug] Saved page 1 PNG: ${page1PngPath}`); + } + } catch (err) { + const durationMs = Date.now() - startTime; + console.error(`[pdfDebug] pdftocairo failed for "${filename}" after ${durationMs}ms:`, err.message); + + metaLog.error = { + stage: 'pdftocairo', + message: err.message, + stack: err.stack, + }; + metaLog.totalDurationMs = durationMs; + metaLog.success = false; + + if (saveToDisk) { + writeDebugFile(`${debugBasePath}-meta.json`, JSON.stringify(metaLog, null, 2)); + } + console.log(`[pdfDebug] ========================================================`); + + return { thumbavif: null, thumbx: null, thumby: null, meta: metaLog }; + } + + // 3. Sharp resize & AVIF conversion + try { + const sharpStartTime = Date.now(); const thumbResult = await sharp(imageBuffer) .resize(THUMB_MAX, THUMB_MAX, { fit: 'inside', withoutEnlargement: true }) .avif({ effort: 4 }) .toBuffer({ resolveWithObject: true }); + const sharpDurationMs = Date.now() - sharpStartTime; + const totalDurationMs = Date.now() - startTime; + + metaLog.sharp = { + thumbx: thumbResult.info.width, + thumby: thumbResult.info.height, + avifBytes: thumbResult.data.length, + durationMs: sharpDurationMs, + }; + metaLog.totalDurationMs = totalDurationMs; + metaLog.success = true; + + console.log(`[pdfDebug] sharp generated AVIF thumbnail ${thumbResult.info.width}x${thumbResult.info.height} (${thumbResult.data.length} bytes) in ${sharpDurationMs}ms (total ${totalDurationMs}ms)`); + + if (saveToDisk) { + const thumbAvifPath = `${debugBasePath}-thumb.avif`; + writeDebugFile(thumbAvifPath, thumbResult.data); + metaLog.savedFiles.thumbAvif = thumbAvifPath; + console.log(`[pdfDebug] Saved AVIF thumbnail: ${thumbAvifPath}`); + + const metaJsonPath = `${debugBasePath}-meta.json`; + writeDebugFile(metaJsonPath, JSON.stringify(metaLog, null, 2)); + console.log(`[pdfDebug] Saved metadata log: ${metaJsonPath}`); + } + + console.log(`[pdfDebug] Successfully completed thumbnail generation for "${filename}"`); + console.log(`[pdfDebug] ========================================================`); + return { thumbavif: thumbResult.data, thumbx: thumbResult.info.width, thumby: thumbResult.info.height, + meta: metaLog, }; } catch (err) { - console.warn('generateDocumentThumb: sharp thumbnail generation failed:', err.message); - return { thumbavif: null, thumbx: null, thumby: null }; + const totalDurationMs = Date.now() - startTime; + console.error(`[pdfDebug] sharp failed for "${filename}" after ${totalDurationMs}ms:`, err.message); + + metaLog.error = { + stage: 'sharp', + message: err.message, + stack: err.stack, + }; + metaLog.totalDurationMs = totalDurationMs; + metaLog.success = false; + + if (saveToDisk) { + writeDebugFile(`${debugBasePath}-meta.json`, JSON.stringify(metaLog, null, 2)); + } + console.log(`[pdfDebug] ========================================================`); + + return { + thumbavif: null, + thumbx: null, + thumby: null, + meta: metaLog, + }; } } + +/** + * Default export compatible with libdashstore's generateDocumentThumb. + */ +export default async function generateDocumentThumb(content, mimetype, options = {}) { + const opts = typeof options === 'object' && options !== null ? options : {}; + const result = await debugGeneratePdfThumbnail(content, { + mimetype, + filename: opts.filename, + debugDir: opts.debugDir, + saveToDisk: opts.saveToDisk ?? true, + }); + return { + thumbavif: result.thumbavif, + thumbx: result.thumbx, + thumby: result.thumby, + }; +}