u

src/api/routes/company.js

@@ -469,6 +469,8 @@ export function register(app, basePath = '') {
      */
     // #region POST /company/import
 
+    const activeImportsByCompany = new Set();
+
     app.post(
         `${basePath}/company/import`,
         uploadCompanyImportFile,
@@ -480,7 +482,13 @@ export function register(app, basePath = '') {
                 return res.status(error.statusCode || 400).json({ error: error.message });
             }
 
-            const importDir = path.resolve(process.cwd(), 'data/import', String(companyId));
+            if (activeImportsByCompany.has(companyId)) {
+                return res.status(409).json({ error: 'Import already in progress for this company' });
+            }
+            activeImportsByCompany.add(companyId);
+
+            try {
+                const importDir = path.resolve(process.cwd(), 'data/import', String(companyId));
 
             // If a file was uploaded, unpack and preserve it
             if (req.file?.buffer) {
@@ -548,18 +556,48 @@ export function register(app, basePath = '') {
                 !req.headers.accept?.includes('application/json');
 
             if (wantsSse) {
-                res.setHeader('Content-Type', 'text/event-stream');
+                req.socket?.setTimeout(0);
-                res.setHeader('Cache-Control', 'no-cache');
+                res.setTimeout?.(0);
-                res.setHeader('Connection', 'keep-alive');
+                req.socket?.setNoDelay(true);
-                res.setHeader('X-Accel-Buffering', 'no');
+
+                res.writeHead(200, {
+                    'Content-Type': 'text/event-stream',
+                    'Cache-Control': 'no-cache, no-transform',
+                    'Connection': 'keep-alive',
+                    'X-Accel-Buffering': 'no',
+                });
                 if (typeof res.flushHeaders === 'function') {
                     res.flushHeaders();
                 }
 
+                // Initial ping to establish frame through any proxy
+                res.write(': ping\n\n');
+
+                req.on('close', () => {
+                    console.warn(`[Company Import SSE] Client request closed (writableEnded: ${res.writableEnded}, destroyed: ${res.destroyed})`);
+                });
+
+                res.on('close', () => {
+                    if (!res.writableEnded) {
+                        console.warn(`[Company Import SSE] Response stream closed (destroyed: ${res.destroyed})`);
+                    }
+                });
+
                 const sseWrite = (event, data) => {
-                    if (res.writableEnded || res.destroyed) return;
+                    if (res.writableEnded || res.destroyed) {
+                        console.warn(`[Company Import SSE DROP] event="${event}" dropped: connection closed (destroyed: ${res.destroyed}, ended: ${res.writableEnded})`);
+                        return;
+                    }
+                    if (event === 'progress') {
+                        console.log(`[Company Import SSE] progress cat=${data.categoriesdone}/${data.categoriestotal} art=${data.articlesdone}/${data.articlestotal} img=${data.imagesdone}/${data.imagestotal} rec=${data.receiptsdone}/${data.receiptstotal}`);
+                    } else if (event === 'images') {
+                        console.log(`[Company Import SSE] images batch count=${Array.isArray(data) ? data.length : 1}`);
+                    } else {
+                        console.log(`[Company Import SSE] event="${event}"`, data);
+                    }
                     res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
                     if (typeof res.flush === 'function') res.flush();
+                    if (typeof req.socket?.uncork === 'function') req.socket.uncork();
                 };
 
                 sseWrite('status', { message: 'Import gestartet. Daten werden verarbeitet...' });
@@ -601,8 +639,11 @@ export function register(app, basePath = '') {
                     res.status(500).json({ error: err.message || 'Import fehlgeschlagen' });
                 }
             }
+        } finally {
+            activeImportsByCompany.delete(companyId);
         }
-    );
+    }
+);
 
     // #endregion
 

src/services/companyImport.js

@@ -47,8 +47,9 @@ const log = createLogger('import');
  * @param {number} opts.googleAuthId
  * @param {number} opts.companyId
  * @param {string} opts.importDir
- * @param {function} [opts.onProgress] - (counts) => void; throttled to 200ms
+ * @param {function} [opts.onProgress] - (counts) => void; throttled to 250ms
  * @param {function} [opts.onImages] - (skelehashes: string[]) => void; batched every 50ms
+ * @param {number} [opts.progressThrottleMs=250] - Progress emission throttle/debounce interval in ms
  * @returns {Promise<object>} Summary outcome with statistics
  */
 export async function executeCompanyImport({
@@ -57,6 +58,7 @@ export async function executeCompanyImport({
     importDir,
     onProgress = () => {},
     onImages = () => {},
+    progressThrottleMs = 250,
 }) {
     const store = await getStore();
 
@@ -105,6 +107,7 @@ export async function executeCompanyImport({
         },
         onProgress,
         onImages,
+        progressThrottleMs,
     });
 
     // Emit baseline counts immediately
@@ -133,12 +136,13 @@ export async function executeCompanyImport({
     });
     // #endregion
 
-    // MARK: - Phase 3: Articles
+    // MARK: - Phase 3: Articles (base data, variants, pfand, pictures)
     // #region Phase 3: Articles
     const articlesResult = await importArticles({
         store,
         googleAuthId,
         includedArticles,
+        articleImageTasks,
         jtlCatIdToDashCatId: categoriesResult.jtlCatIdToDashCatId,
         hauptDashId: categoriesResult.hauptDashId,
         brandNameToId: brandsResult.brandNameToId,
@@ -146,44 +150,30 @@ export async function executeCompanyImport({
     });
     // #endregion
 
-    // MARK: - Phase 4: Variants
+    // MARK: - Phase 4: Orders / Receipts
-    // #region Phase 4: Variants
+    // #region Phase 4: Orders / Receipts
-    const variantsResult = await importVariants({
-        store,
-        googleAuthId,
-        includedArticles,
-        jtlArtIdToSku: articlesResult.jtlArtIdToSku,
-    });
-    // #endregion
-
-    // MARK: - Phase 5: Pfand / Deposit Options
-    // #region Phase 5: Pfand
-    const pfandResult = await importPfand({
-        store,
-        googleAuthId,
-        includedArticles,
-        hauptDashId: categoriesResult.hauptDashId,
-        parentSkus: variantsResult.parentSkus,
-    });
-    // #endregion
-
-    // MARK: - Phase 6: Article Pictures
-    // #region Phase 6: Article Pictures
-    const imagesResult = await importArticleImages({
-        store,
-        googleAuthId,
-        articleImageTasks,
-        progressTracker,
-    });
-    // #endregion
-
-    // MARK: - Phase 7: Orders / Receipts
-    // #region Phase 7: Orders / Receipts
     const allArtIdToSku = new Map();
+    const parentSkus = new Set(articlesResult.parentSkus || []);
+    const parentIds = new Set();
     for (const a of artRecords) {
         const kArt = cleanValue(a.id ?? a.kArtikel);
         const sku = cleanValue(a.sku ?? a.cArtNr);
         if (kArt && sku) allArtIdToSku.set(kArt, sku);
+
+        const pid = cleanValue(a.parentid ?? a.kVaterArtikel);
+        if (pid && pid !== '0') parentIds.add(pid);
+        const isParent = a.is_parent === '1' || a.nIstVater === '1';
+        if (isParent) {
+            if (kArt) parentIds.add(kArt);
+            if (sku) parentSkus.add(sku);
+        }
+    }
+    for (const a of artRecords) {
+        const kArt = cleanValue(a.id ?? a.kArtikel);
+        if (kArt && parentIds.has(kArt)) {
+            const sku = cleanValue(a.sku ?? a.cArtNr);
+            if (sku) parentSkus.add(sku);
+        }
     }
 
     const ordersResult = await importOrders({
@@ -192,6 +182,7 @@ export async function executeCompanyImport({
         manRecords,
         itemRecords,
         jtlArtIdToSku: allArtIdToSku,
+        parentSkus,
         progressTracker,
     });
     // #endregion
@@ -213,11 +204,11 @@ export async function executeCompanyImport({
             articlesTotal: includedArticles.length,
             articlesImported: articlesResult.articlesImported,
             articlesUpdated: articlesResult.articlesUpdated,
-            variantsLinked: variantsResult.variantsLinked,
+            variantsLinked: articlesResult.variantsLinked,
-            pfandArticlesCreated: pfandResult.pfandArticlesCreated,
+            pfandArticlesCreated: articlesResult.pfandArticlesCreated,
-            pfandArticlesUpdated: pfandResult.pfandArticlesUpdated,
+            pfandArticlesUpdated: articlesResult.pfandArticlesUpdated,
-            pfandOptionsLinked: pfandResult.pfandOptionsLinked,
+            pfandOptionsLinked: articlesResult.pfandOptionsLinked,
-            articlePictures: imagesResult.totalImagesImported,
+            articlePictures: articlesResult.totalImagesImported,
             ordersTotal: totalReceiptsCount,
             ordersImported: ordersResult.ordersImported,
             ordersSkipped: ordersResult.ordersSkipped,

src/services/companyImport/articleImporter.js

@@ -1,6 +1,10 @@
 // MARK: - Imports
 // #region Imports
+import fs from 'fs';
 import { cleanValue, parseGermanFloat } from './csvParser.js';
+import { setupVariantAttributes } from './variantImporter.js';
+import { setupPfandArticles, linkArticlePfand, extractPfandValue } from './pfandImporter.js';
+import { computeSkeleHash } from './imageHasher.js';
 import { createLogger } from '../../util/logger.js';
 // #endregion
 
@@ -10,32 +14,120 @@ const log = createLogger('import');
 // #region Article Importer
 
 /**
- * Imports articles (insert or update), setting prices, categories, barcodes, and brands.
+ * Imports articles (insert or update), setting prices, categories, barcodes, brands,
+ * variants, pfand deposit options, and pictures in a single unified loop.
  *
  * @param {object} params
  * @param {object} params.store - Store singleton
  * @param {number} params.googleAuthId
  * @param {Array<Record<string, string>>} params.includedArticles
+ * @param {Array<object>} [params.articleImageTasks=[]]
  * @param {Map<string, number>} params.jtlCatIdToDashCatId
  * @param {number} params.hauptDashId
  * @param {Map<string, number>} params.brandNameToId
  * @param {object} params.progressTracker
- * @returns {Promise<{ jtlArtIdToSku: Map<string, string>, articlesImported: number, articlesUpdated: number }>}
+ * @returns {Promise<{
+ *   jtlArtIdToSku: Map<string, string>,
+ *   articlesImported: number,
+ *   articlesUpdated: number,
+ *   variantsLinked: number,
+ *   parentsCount: number,
+ *   parentSkus: Set<string>,
+ *   pfandArticlesCreated: number,
+ *   pfandArticlesUpdated: number,
+ *   pfandOptionsLinked: number,
+ *   totalImagesImported: number
+ * }>}
  */
 export async function importArticles({
     store,
     googleAuthId,
     includedArticles,
+    articleImageTasks = [],
     jtlCatIdToDashCatId,
     hauptDashId,
     brandNameToId,
     progressTracker,
 }) {
+    // 1. Group article image tasks by SKU
+    const articleImagesBySku = new Map();
+    for (const task of articleImageTasks) {
+        if (!task.sku) continue;
+        if (!articleImagesBySku.has(task.sku)) {
+            articleImagesBySku.set(task.sku, []);
+        }
+        articleImagesBySku.get(task.sku).push(task);
+    }
+
+    // 2. Setup Pfand category and Pfand articles upfront (e.g. PFAND-0.08, PFAND-0.15, etc.)
+    const pfandSetup = await setupPfandArticles({
+        store,
+        googleAuthId,
+        hauptDashId,
+        includedArticles,
+    });
+
+    // 3. Setup variant attributes resolver
+    const variantSetup = await setupVariantAttributes({ store, googleAuthId });
+
+    // 4. Pre-scan articles to identify parents and map parent Pfand / parent rows
+    const parentJtlIds = new Set();
+    const parentPfandMap = new Map();
+    const parentRowMap = new Map();
     const jtlArtIdToSku = new Map();
-    let articlesImported = 0;
-    let articlesUpdated = 0;
 
     for (const a of includedArticles) {
+        const id = cleanValue(a.id ?? a.kArtikel);
+        const pid = cleanValue(a.parentid ?? a.kVaterArtikel);
+        const sku = cleanValue(a.sku ?? a.cArtNr);
+        const isParent = a.is_parent === '1' || a.nIstVater === '1';
+
+        if (id && sku) {
+            jtlArtIdToSku.set(id, sku);
+        }
+        if (pid && pid !== '0') {
+            parentJtlIds.add(pid);
+        }
+        if (isParent && id) {
+            parentJtlIds.add(id);
+        }
+    }
+
+    // Identify parent rows and their Pfand values
+    for (const a of includedArticles) {
+        const id = cleanValue(a.id ?? a.kArtikel);
+        if (id && parentJtlIds.has(id)) {
+            parentRowMap.set(id, a);
+            const val = extractPfandValue(a.attributes);
+            if (val != null) {
+                parentPfandMap.set(id, val);
+            }
+        }
+    }
+
+    // 5. Sort articles so parents come first, followed by children and standalone articles
+    const sortedArticles = [...includedArticles].sort((a, b) => {
+        const aId = cleanValue(a.id ?? a.kArtikel);
+        const bId = cleanValue(b.id ?? b.kArtikel);
+        const aIsParent = parentJtlIds.has(aId);
+        const bIsParent = parentJtlIds.has(bId);
+        if (aIsParent && !bIsParent) return -1;
+        if (!aIsParent && bIsParent) return 1;
+        return 0;
+    });
+
+    const parentSkus = new Set();
+    const parentTargetAttrIds = new Map(); // parentSku -> targetAttrId
+    const parentUsedValues = new Map(); // parentSku -> Set<lowercaseValue>
+
+    let articlesImported = 0;
+    let articlesUpdated = 0;
+    let variantsLinked = 0;
+    let pfandOptionsLinked = 0;
+    let articlePicturesImported = 0;
+
+    // 6. Unified single loop over all articles
+    for (const a of sortedArticles) {
         const jtlArtId = String(a.id ?? a.kArtikel);
         const sku = cleanValue(a.sku ?? a.cArtNr);
         if (!sku) {
@@ -45,6 +137,12 @@ export async function importArticles({
 
         jtlArtIdToSku.set(jtlArtId, sku);
 
+        const isParent = parentJtlIds.has(jtlArtId);
+        if (isParent) {
+            parentSkus.add(sku);
+        }
+
+        // --- Basic article fields ---
         let name = cleanValue(a.name ?? a.cName);
         let desc = cleanValue(a.desc ?? a.descriptions);
 
@@ -174,17 +272,116 @@ export async function importArticles({
             log.warn(`Failed to save article ${sku} ("${name}"):`, err.message);
         }
 
+        // --- Variants handling ---
+        if (isParent) {
+            const rawAttrName = cleanValue(a.variation ?? a.kEigenschaftKombi);
+            const targetAttrId = await variantSetup.resolveAttributeId(rawAttrName || 'Variante');
+            if (targetAttrId) {
+                parentTargetAttrIds.set(sku, targetAttrId);
+                try {
+                    await store.article.toggleParent(googleAuthId, sku, {
+                        isParent: true,
+                        attributeNameId: targetAttrId,
+                    });
+                } catch (err) {
+                    log.warn(`Could not toggle parent for ${sku}:`, err.message);
+                }
+            }
+        } else {
+            const parentJtlId = cleanValue(a.parentid ?? a.kVaterArtikel);
+            if (parentJtlId && parentJtlId !== '0') {
+                const parentSku = jtlArtIdToSku.get(parentJtlId);
+                if (parentSku && parentSku !== sku) {
+                    const targetAttrId = parentTargetAttrIds.get(parentSku);
+                    if (!parentUsedValues.has(parentSku)) {
+                        parentUsedValues.set(parentSku, new Set());
+                    }
+                    const usedSet = parentUsedValues.get(parentSku);
+
+                    let val = cleanValue(a.variation ?? a.kEigenschaftKombi) || 'Standard';
+                    if (usedSet.has(val.toLowerCase())) {
+                        val = `${val} (${sku})`;
+                    }
+                    usedSet.add(val.toLowerCase());
+
+                    try {
+                        if (targetAttrId) {
+                            await store.article.attachChild(googleAuthId, parentSku, {
+                                sku,
+                                value: val,
+                                reparent: true,
+                            });
+                            variantsLinked++;
+                        } else {
+                            await store.article.setParentBySku(googleAuthId, sku, parentSku);
+                            variantsLinked++;
+                        }
+                    } catch (err) {
+                        log.warn(`Failed to attach child ${sku} to parent ${parentSku}:`, err.message);
+                    }
+                }
+            }
+        }
+
+        // --- Pfand handling (only for non-parents) ---
+        if (!isParent) {
+            const parentJtlId = cleanValue(a.parentid ?? a.kVaterArtikel);
+            let pfandVal = extractPfandValue(a.attributes);
+            if (pfandVal == null && parentJtlId && parentJtlId !== '0') {
+                if (parentPfandMap.has(parentJtlId)) {
+                    pfandVal = parentPfandMap.get(parentJtlId);
+                }
+            }
+
+            if (pfandVal != null) {
+                const linked = await linkArticlePfand({
+                    store,
+                    googleAuthId,
+                    sku,
+                    pfandVal,
+                });
+                if (linked) pfandOptionsLinked++;
+            }
+        }
+
+        // --- Pictures handling for this article ---
+        const imagesForThisArt = articleImagesBySku.get(sku);
+        if (imagesForThisArt && imagesForThisArt.length > 0) {
+            for (const imgTask of imagesForThisArt) {
+                try {
+                    const imgBuf = fs.readFileSync(imgTask.fullPath);
+                    await store.article.addImage(googleAuthId, sku, imgBuf);
+                    articlePicturesImported++;
+
+                    const skele = await computeSkeleHash(imgBuf);
+                    progressTracker.emitSkeleHash(skele?.base62 || null);
+                } catch (imgErr) {
+                    log.warn(`Article image add failed for ${sku} (${imgTask.relPath}):`, imgErr.message);
+                }
+                progressTracker.counts.imagesdone++;
+                progressTracker.emitProgress();
+            }
+        }
+
         progressTracker.counts.articlesdone++;
         progressTracker.emitProgress();
     }
 
+    progressTracker.flushSkeleHashes();
     progressTracker.emitProgress(true);
-    log.info(`Articles complete: New: ${articlesImported}, Updated: ${articlesUpdated}`);
+    log.info(`Articles complete: New: ${articlesImported}, Updated: ${articlesUpdated}, Variants: ${variantsLinked}, Pfand: ${pfandOptionsLinked}, Pictures: ${articlePicturesImported}`);
 
     return {
         jtlArtIdToSku,
         articlesImported,
         articlesUpdated,
+        variantsLinked,
+        parentsCount: parentSkus.size,
+        parentSkus,
+        pfandArticlesCreated: pfandSetup.pfandArticlesCreated,
+        pfandArticlesUpdated: pfandSetup.pfandArticlesUpdated,
+        pfandOptionsLinked,
+        totalImagesImported: articlePicturesImported,
     };
 }
 

src/services/companyImport/orderImporter.js

@@ -18,6 +18,7 @@ const log = createLogger('import');
  * @param {Array<Record<string, string>>} params.manRecords
  * @param {Array<Record<string, string>>} params.itemRecords
  * @param {Map<string, string>} [params.jtlArtIdToSku]
+ * @param {Set<string>} [params.parentSkus]
  * @param {object} params.progressTracker
  * @returns {Promise<{ ordersImported: number, ordersSkipped: number, paymentsImported: number, shippingsImported: number }>}
  */
@@ -27,6 +28,7 @@ export async function importOrders({
     manRecords,
     itemRecords,
     jtlArtIdToSku = new Map(),
+    parentSkus = new Set(),
     progressTracker,
 }) {
     let ordersImported = 0;
@@ -135,13 +137,13 @@ export async function importOrders({
         }
     };
 
-    for (const m of manRecords) {
+    const processManifest = async (m) => {
         const orderNumber = cleanValue(m.nr ?? m.cAuftragsNr);
         if (!orderNumber || existingOrderNumbers.has(orderNumber)) {
             ordersSkipped++;
             progressTracker.counts.receiptsdone++;
             progressTracker.emitProgress();
-            continue;
+            return;
         }
 
         const manifestKey = String(m.id ?? m.kAuftrag ?? '').trim();
@@ -150,14 +152,17 @@ export async function importOrders({
             ordersSkipped++;
             progressTracker.counts.receiptsdone++;
             progressTracker.emitProgress();
-            continue;
+            return;
         }
 
         rawItems.sort((a, b) => (Number(a.sort ?? a.nSort) || 0) - (Number(b.sort ?? b.nSort) || 0));
 
         const lines = rawItems.map((it) => {
             const articleId = cleanValue(it.articleid ?? it.kArtikel);
-            const sku = cleanValue(it.sku ?? it.cArtNr) || (articleId && jtlArtIdToSku?.get?.(articleId)) || null;
+            let sku = cleanValue(it.sku ?? it.cArtNr) || (articleId && jtlArtIdToSku?.get?.(articleId)) || null;
+            if (sku && parentSkus.has(sku)) {
+                sku = null;
+            }
             const rawName = cleanValue(it.name ?? it.cName ?? it.cNameStandard);
             const name = rawName || (sku ? `Artikel ${sku}` : 'Artikel');
             const amount = parseGermanFloat(it.amount ?? it.fAnzahl, 1);
@@ -198,7 +203,30 @@ export async function importOrders({
             existingOrderNumbers.add(orderNumber);
             ordersImported++;
         } catch (err) {
-            log.warn(`Failed to create order ${orderNumber}:`, err.message);
+            const parentMatch = err.message?.match(/A parent article cannot be booked on documents \(([^)]+)\)/);
+            if (parentMatch) {
+                const offendingSku = parentMatch[1];
+                parentSkus.add(offendingSku);
+                for (const l of lines) {
+                    if (l.sku === offendingSku) l.sku = null;
+                }
+                try {
+                    createdOrder = await store.order.create(googleAuthId, {
+                        number: orderNumber,
+                        externalNumber: cleanValue(m.extnr ?? m.cExterneAuftragsnummer) || null,
+                        date: validDate,
+                        createdAt: validDate,
+                        entityId: entityId || undefined,
+                        lines,
+                    });
+                    existingOrderNumbers.add(orderNumber);
+                    ordersImported++;
+                } catch (retryErr) {
+                    log.warn(`Failed to create order ${orderNumber} after stripping parent SKU ${offendingSku}:`, retryErr.message);
+                }
+            } else {
+                log.warn(`Failed to create order ${orderNumber}:`, err.message);
+            }
         }
 
         if (createdOrder) {
@@ -242,25 +270,31 @@ export async function importOrders({
 
                 if (shippableLines.length > 0) {
                     try {
+                        const isSelbstabholer = (shippingType || '').trim().toLowerCase() === 'selbstabholer';
+                        const deliveryState = isShipped ? 'delivered' : 'new';
+
                         const shipment = await store.shipment.create(googleAuthId, {
                             orderId: createdOrder.id,
                             date: validDate,
+                            state: deliveryState,
                             lines: shippableLines,
                         });
 
-                        const trackingCode = cleanValue(m.trackingCode ?? m.tracking_code ?? m.cTracking);
+                        if (!isSelbstabholer) {
-                        const carrier = cleanValue(m.carrier ?? m.cCarrier);
+                            const trackingCode = cleanValue(m.trackingCode ?? m.tracking_code ?? m.cTracking);
-                        const shippingState = isShipped ? 'shipped' : 'draft';
+                            const carrier = cleanValue(m.carrier ?? m.cCarrier);
+                            const shippingState = isShipped ? 'shipped' : 'draft';
 
-                        await store.shipping.create(googleAuthId, {
+                            await store.shipping.create(googleAuthId, {
-                            manifestId: shipment.id,
+                                manifestId: shipment.id,
-                            type: shippingType || 'standard',
+                                type: shippingType || 'standard',
-                            state: shippingState,
+                                state: shippingState,
-                            trackingCode: trackingCode || undefined,
+                                trackingCode: trackingCode || undefined,
-                            carrier: carrier || undefined,
+                                carrier: carrier || undefined,
-                            warehouseId: defaultWarehouseId || undefined,
+                                warehouseId: defaultWarehouseId || undefined,
-                        });
+                            });
-                        shippingsImported++;
+                            shippingsImported++;
+                        }
                     } catch (err) {
                         log.warn(`Failed to create shipment/shipping for order ${orderNumber}:`, err.message);
                     }
@@ -270,6 +304,40 @@ export async function importOrders({
 
         progressTracker.counts.receiptsdone++;
         progressTracker.emitProgress();
+    };
+
+    const runTx = async (fn) => {
+        if (typeof store.transaction === 'function') {
+            return store.transaction(fn);
+        }
+        if (typeof store.provider?.transaction === 'function') {
+            return store.provider.transaction(fn);
+        }
+        return fn();
+    };
+
+    const BATCH_SIZE = 25;
+    for (let i = 0; i < manRecords.length; i += BATCH_SIZE) {
+        const batch = manRecords.slice(i, i + BATCH_SIZE);
+        try {
+            await runTx(async () => {
+                for (const m of batch) {
+                    await processManifest(m);
+                }
+            });
+        } catch (batchErr) {
+            log.warn(`Batch of ${batch.length} manifests failed in transaction, falling back to one-by-one:`, batchErr.message);
+            for (const m of batch) {
+                try {
+                    await processManifest(m);
+                } catch (singleErr) {
+                    log.warn(`Single manifest failed:`, singleErr.message);
+                    progressTracker.counts.receiptsdone++;
+                    progressTracker.emitProgress();
+                }
+            }
+        }
+        await new Promise((resolve) => setImmediate(resolve));
     }
 
     progressTracker.emitProgress(true);

src/services/companyImport/pfandImporter.js

@@ -48,6 +48,180 @@ export function formatPfandName(val) {
 
 // #endregion
 
+// MARK: - Pfand Setup & Linking Helpers
+// #region Pfand Setup & Linking Helpers
+
+/**
+ * Pre-creates the Pfand category and Pfand articles upfront.
+ *
+ * @param {object} params
+ * @param {object} params.store - Store singleton
+ * @param {number} params.googleAuthId
+ * @param {number} params.hauptDashId
+ * @param {Array<Record<string, string>>} params.includedArticles
+ * @returns {Promise<{ pfandArticlesCreated: number, pfandArticlesUpdated: number, pfandCatId: number|null, uniqueValues: number[] }>}
+ */
+export async function setupPfandArticles({
+    store,
+    googleAuthId,
+    hauptDashId,
+    includedArticles,
+}) {
+    // 1. Discover unique Pfand values from all included articles (checking attributes)
+    const uniqueValuesSet = new Set();
+    for (const a of includedArticles) {
+        const val = extractPfandValue(a.attributes);
+        if (val != null) {
+            uniqueValuesSet.add(val);
+        }
+    }
+
+    if (uniqueValuesSet.size === 0) {
+        log.info('No pfand attributes detected among articles.');
+        return {
+            pfandArticlesCreated: 0,
+            pfandArticlesUpdated: 0,
+            pfandCatId: null,
+            uniqueValues: [],
+        };
+    }
+
+    const uniqueValues = Array.from(uniqueValuesSet).sort((a, b) => a - b);
+    log.info(`Discovered ${uniqueValues.length} unique Pfand values: ${uniqueValues.join(', ')}`);
+
+    // 2. Resolve or create "Pfand" subcategory under "Haupt"
+    let pfandCatId = null;
+    try {
+        const treeResult = await store.category.getTreeForCompany(googleAuthId);
+        const existingTree = treeResult.tree || [];
+        const existingPfandCat = existingTree.find(
+            (c) => c.parentId === hauptDashId && String(c.name || '').trim().toLowerCase() === 'pfand'
+        );
+
+        if (existingPfandCat) {
+            pfandCatId = existingPfandCat.id;
+        } else {
+            const created = await store.category.createUnderParent({
+                googleAuthId,
+                parentId: hauptDashId,
+                name: 'Pfand',
+            });
+            pfandCatId = created.id;
+            log.info(`Created "Pfand" category under Haupt (id: ${pfandCatId})`);
+        }
+    } catch (err) {
+        log.warn('Could not resolve or create Pfand category under Haupt, falling back to Haupt:', err.message);
+        pfandCatId = hauptDashId;
+    }
+
+    // 3. Create or update Pfand articles
+    let pfandArticlesCreated = 0;
+    let pfandArticlesUpdated = 0;
+
+    for (const val of uniqueValues) {
+        const pfandSku = formatPfandSku(val);
+        const pfandName = formatPfandName(val);
+
+        let existingArticle = null;
+        try {
+            const res = await store.article.getBySku(googleAuthId, pfandSku);
+            existingArticle = res.article;
+        } catch {
+            existingArticle = null;
+        }
+
+        try {
+            if (existingArticle) {
+                await store.article.updateBySku(googleAuthId, pfandSku, {
+                    name: pfandName,
+                    track_stock: 0,
+                });
+                await store.article.setPricesBySku(googleAuthId, pfandSku, [
+                    { gross: val, tax: 19 },
+                ]);
+                pfandArticlesUpdated++;
+                log.info(`Updated existing Pfand article: ${pfandSku} ("${pfandName}")`);
+            } else {
+                await store.article.createUnderParent({
+                    googleAuthId,
+                    parentId: pfandCatId,
+                    name: pfandName,
+                    sku: pfandSku,
+                    track_stock: 0,
+                });
+                await store.article.setPricesBySku(googleAuthId, pfandSku, [
+                    { gross: val, tax: 19 },
+                ]);
+                pfandArticlesCreated++;
+                log.info(`Created new Pfand article: ${pfandSku} ("${pfandName}")`);
+            }
+        } catch (err) {
+            log.warn(`Failed to create or update Pfand article ${pfandSku}:`, err.message);
+        }
+    }
+
+    return {
+        pfandArticlesCreated,
+        pfandArticlesUpdated,
+        pfandCatId,
+        uniqueValues,
+    };
+}
+
+/**
+ * Links an article to its mandatory Pfand deposit option.
+ *
+ * @param {object} params
+ * @param {object} params.store
+ * @param {number} params.googleAuthId
+ * @param {string} params.sku
+ * @param {number} params.pfandVal
+ * @returns {Promise<boolean>}
+ */
+export async function linkArticlePfand({
+    store,
+    googleAuthId,
+    sku,
+    pfandVal,
+}) {
+    if (!sku || pfandVal == null) return false;
+    const pfandSku = formatPfandSku(pfandVal);
+
+    let existingOptions = [];
+    try {
+        existingOptions = (await store.article.getOptionslist(googleAuthId, sku)) || [];
+    } catch {
+        existingOptions = [];
+    }
+
+    const otherOptions = existingOptions.filter((o) => !o.sku?.startsWith('PFAND-'));
+    const newOptions = [
+        ...otherOptions.map((o, idx) => ({
+            sku: o.sku,
+            articleId: o.articleId,
+            count: o.count || 1,
+            sort: idx,
+            mandated: Boolean(o.mandated),
+        })),
+        {
+            sku: pfandSku,
+            count: 1,
+            sort: otherOptions.length,
+            mandated: true,
+        },
+    ];
+
+    try {
+        await store.article.setOptionslist(googleAuthId, sku, newOptions);
+        return true;
+    } catch (err) {
+        log.warn(`Failed to link Pfand option ${pfandSku} to article ${sku}:`, err.message);
+        return false;
+    }
+}
+
+// #endregion
+
 // MARK: - Pfand Importer
 // #region Pfand Importer
 
@@ -116,125 +290,31 @@ export async function importPfand({
         };
     }
 
-    // 3. Find unique Pfand values
+    // 3. Setup Pfand category and Pfand articles
-    const uniqueValues = Array.from(new Set(articleTargets.map((t) => t.pfandVal))).sort((a, b) => a - b);
+    const setupResult = await setupPfandArticles({
-    log.info(`Discovered ${articleTargets.length} articles with Pfand across ${uniqueValues.length} unique values: ${uniqueValues.join(', ')}`);
+        store,
+        googleAuthId,
+        hauptDashId,
+        includedArticles,
+    });
 
-    // 4. Resolve or create "Pfand" subcategory under "Haupt"
+    // 4. Link each target article to its Pfand article as a mandatory option
-    let pfandCatId = null;
-    try {
-        const treeResult = await store.category.getTreeForCompany(googleAuthId);
-        const existingTree = treeResult.tree || [];
-        const existingPfandCat = existingTree.find(
-            (c) => c.parentId === hauptDashId && String(c.name || '').trim().toLowerCase() === 'pfand'
-        );
-
-        if (existingPfandCat) {
-            pfandCatId = existingPfandCat.id;
-        } else {
-            const created = await store.category.createUnderParent({
-                googleAuthId,
-                parentId: hauptDashId,
-                name: 'Pfand',
-            });
-            pfandCatId = created.id;
-            log.info(`Created "Pfand" category under Haupt (id: ${pfandCatId})`);
-        }
-    } catch (err) {
-        log.warn('Could not resolve or create Pfand category under Haupt, falling back to Haupt:', err.message);
-        pfandCatId = hauptDashId;
-    }
-
-    // 5. Create or update Pfand articles
-    let pfandArticlesCreated = 0;
-    let pfandArticlesUpdated = 0;
-
-    for (const val of uniqueValues) {
-        const pfandSku = formatPfandSku(val);
-        const pfandName = formatPfandName(val);
-
-        let existingArticle = null;
-        try {
-            const res = await store.article.getBySku(googleAuthId, pfandSku);
-            existingArticle = res.article;
-        } catch {
-            existingArticle = null;
-        }
-
-        try {
-            if (existingArticle) {
-                await store.article.updateBySku(googleAuthId, pfandSku, {
-                    name: pfandName,
-                    track_stock: 0,
-                });
-                await store.article.setPricesBySku(googleAuthId, pfandSku, [
-                    { gross: val, tax: 19 },
-                ]);
-                pfandArticlesUpdated++;
-                log.info(`Updated existing Pfand article: ${pfandSku} ("${pfandName}")`);
-            } else {
-                await store.article.createUnderParent({
-                    googleAuthId,
-                    parentId: pfandCatId,
-                    name: pfandName,
-                    sku: pfandSku,
-                    track_stock: 0,
-                });
-                await store.article.setPricesBySku(googleAuthId, pfandSku, [
-                    { gross: val, tax: 19 },
-                ]);
-                pfandArticlesCreated++;
-                log.info(`Created new Pfand article: ${pfandSku} ("${pfandName}")`);
-            }
-        } catch (err) {
-            log.warn(`Failed to create or update Pfand article ${pfandSku}:`, err.message);
-        }
-    }
-
-    // 6. Link each target article to its Pfand article as a mandatory option
     let pfandOptionsLinked = 0;
-
     for (const { sku, pfandVal } of articleTargets) {
-        const pfandSku = formatPfandSku(pfandVal);
+        const linked = await linkArticlePfand({
-
+            store,
-        let existingOptions = [];
+            googleAuthId,
-        try {
+            sku,
-            existingOptions = (await store.article.getOptionslist(googleAuthId, sku)) || [];
+            pfandVal,
-        } catch {
+        });
-            existingOptions = [];
+        if (linked) pfandOptionsLinked++;
-        }
-
-        // Filter out any existing PFAND options to prevent duplicates, keep any other options
-        const otherOptions = existingOptions.filter((o) => !o.sku?.startsWith('PFAND-'));
-        const newOptions = [
-            ...otherOptions.map((o, idx) => ({
-                sku: o.sku,
-                articleId: o.articleId,
-                count: o.count || 1,
-                sort: idx,
-                mandated: Boolean(o.mandated),
-            })),
-            {
-                sku: pfandSku,
-                count: 1,
-                sort: otherOptions.length,
-                mandated: true,
-            },
-        ];
-
-        try {
-            await store.article.setOptionslist(googleAuthId, sku, newOptions);
-            pfandOptionsLinked++;
-        } catch (err) {
-            log.warn(`Failed to link Pfand option ${pfandSku} to article ${sku}:`, err.message);
-        }
     }
 
-    log.info(`Pfand import complete: ${pfandArticlesCreated} created, ${pfandArticlesUpdated} updated, ${pfandOptionsLinked} linked as mandatory options.`);
+    log.info(`Pfand import complete: ${setupResult.pfandArticlesCreated} created, ${setupResult.pfandArticlesUpdated} updated, ${pfandOptionsLinked} linked as mandatory options.`);
 
     return {
-        pfandArticlesCreated,
+        pfandArticlesCreated: setupResult.pfandArticlesCreated,
-        pfandArticlesUpdated,
+        pfandArticlesUpdated: setupResult.pfandArticlesUpdated,
         pfandOptionsLinked,
     };
 }

src/services/companyImport/progressTracker.js

@@ -16,7 +16,7 @@ export function createProgressTracker({
     initialCounts,
     onProgress = () => {},
     onImages = () => {},
-    progressThrottleMs = 200,
+    progressThrottleMs = 250,
     imageBatchMs = 50,
 }) {
     const counts = {
@@ -32,11 +32,33 @@ export function createProgressTracker({
     };
 
     let lastProgressEmit = 0;
+    let progressTimer = null;
+
     const emitProgress = (force = false) => {
         const now = Date.now();
-        if (force || now - lastProgressEmit >= progressThrottleMs) {
+        if (force) {
+            if (progressTimer) {
+                clearTimeout(progressTimer);
+                progressTimer = null;
+            }
             lastProgressEmit = now;
             onProgress({ ...counts });
+            return;
+        }
+
+        if (now - lastProgressEmit >= progressThrottleMs) {
+            if (progressTimer) {
+                clearTimeout(progressTimer);
+                progressTimer = null;
+            }
+            lastProgressEmit = now;
+            onProgress({ ...counts });
+        } else if (!progressTimer) {
+            progressTimer = setTimeout(() => {
+                progressTimer = null;
+                lastProgressEmit = Date.now();
+                onProgress({ ...counts });
+            }, progressThrottleMs);
         }
     };
 

src/services/companyImport/variantImporter.js

@@ -7,25 +7,18 @@ import { createLogger } from '../../util/logger.js';
 
 const log = createLogger('import');
 
-// MARK: - Variant Importer
+// MARK: - Variant Setup Helper
-// #region Variant Importer
+// #region Variant Setup Helper
 
 /**
- * Resolves attribute definitions and links variant child articles to parents.
+ * Initializes attribute definitions and provides a cached resolver for attribute IDs.
  *
  * @param {object} params
- * @param {object} params.store - Store singleton
+ * @param {object} params.store
  * @param {number} params.googleAuthId
- * @param {Array<Record<string, string>>} params.includedArticles
+ * @returns {Promise<{ attributeMap: Map<string, number>, resolveAttributeId: (rawName: string) => Promise<number|null> }>}
- * @param {Map<string, string>} params.jtlArtIdToSku
- * @returns {Promise<{ variantsLinked: number, parentsCount: number }>}
  */
-export async function importVariants({
+export async function setupVariantAttributes({ store, googleAuthId }) {
-    store,
-    googleAuthId,
-    includedArticles,
-    jtlArtIdToSku,
-}) {
     const attributeMap = new Map();
     try {
         const attrList = await store.attributename.list(googleAuthId);
@@ -57,6 +50,35 @@ export async function importVariants({
         }
     };
 
+    return {
+        attributeMap,
+        resolveAttributeId,
+    };
+}
+
+// #endregion
+
+// MARK: - Variant Importer
+// #region Variant Importer
+
+/**
+ * Resolves attribute definitions and links variant child articles to parents.
+ *
+ * @param {object} params
+ * @param {object} params.store - Store singleton
+ * @param {number} params.googleAuthId
+ * @param {Array<Record<string, string>>} params.includedArticles
+ * @param {Map<string, string>} params.jtlArtIdToSku
+ * @returns {Promise<{ variantsLinked: number, parentsCount: number }>}
+ */
+export async function importVariants({
+    store,
+    googleAuthId,
+    includedArticles,
+    jtlArtIdToSku,
+}) {
+    const { resolveAttributeId } = await setupVariantAttributes({ store, googleAuthId });
+
     // Group children by parent SKU
     const parentChildrenMap = new Map();
     for (const child of includedArticles) {