From 1eb5cb4e592ed5ff29ff2ea1580bdf9526f02d3e Mon Sep 17 00:00:00 2001 From: seb Date: Sun, 11 Oct 2026 18:11:55 +0200 Subject: [PATCH] u --- src/services/companyImport.js | 812 +++++++++++++++------------------- 1 file changed, 348 insertions(+), 464 deletions(-) diff --git a/src/services/companyImport.js b/src/services/companyImport.js index 3a9ef23..2bf8dc5 100644 --- a/src/services/companyImport.js +++ b/src/services/companyImport.js @@ -36,13 +36,51 @@ async function computeSkeleHash(buffer) { } /** - * Parses RFC 4180 CSV text into an array of objects keyed by header. + * Cleans formula prefixes (="..."), quotes, and whitespace from CSV values. + * + * @param {string|any} str + * @returns {string} + */ +export function cleanValue(str) { + if (str == null) return ''; + let val = String(str).trim(); + if (val.startsWith('=')) { + val = val.slice(1).trim(); + } + if (val.startsWith('"') && val.endsWith('"')) { + val = val.slice(1, -1).trim(); + } + return val; +} + +/** + * Parses German decimal format numbers (e.g. "8,36" -> 8.36). + * + * @param {string|number} str + * @param {number} [fallback=0] + * @returns {number} + */ +export function parseGermanFloat(str, fallback = 0) { + const clean = cleanValue(str).replace(',', '.'); + const num = parseFloat(clean); + return Number.isNaN(num) ? fallback : num; +} + +/** + * Parses CSV text into an array of objects keyed by header. + * Supports auto-detecting semicolon (;) vs comma (,) delimiters, + * quoted fields with embedded newlines/quotes, and formula prefixes. * * @param {string} text - Raw CSV text * @returns {Array>} */ export function parseCsvRecords(text) { const clean = text.replace(/^\uFEFF/, ''); + const firstLine = clean.split(/[\r\n]/)[0] || ''; + const semiCount = (firstLine.match(/;/g) || []).length; + const commaCount = (firstLine.match(/,/g) || []).length; + const delimiter = semiCount >= commaCount ? ';' : ','; + const rows = []; let field = ''; let row = []; @@ -73,8 +111,8 @@ export function parseCsvRecords(text) { inQuotes = true; i++; continue; - } else if (c === ',') { - row.push(field); + } else if (c === delimiter) { + row.push(cleanValue(field)); field = ''; i++; continue; @@ -82,14 +120,14 @@ export function parseCsvRecords(text) { if (i + 1 < len && clean[i + 1] === '\n') { i++; } - row.push(field); + row.push(cleanValue(field)); rows.push(row); field = ''; row = []; i++; continue; } else if (c === '\n') { - row.push(field); + row.push(cleanValue(field)); rows.push(row); field = ''; row = []; @@ -103,11 +141,11 @@ export function parseCsvRecords(text) { } } if (field !== '' || row.length > 0) { - row.push(field); + row.push(cleanValue(field)); rows.push(row); } if (rows.length === 0) return []; - const header = rows[0].map((h) => h.trim()); + const header = rows[0].map((h) => cleanValue(h)); const records = []; for (let r = 1; r < rows.length; r++) { const curr = rows[r]; @@ -122,7 +160,7 @@ export function parseCsvRecords(text) { } /** - * Normalizes common variant attribute names and typos from JTL exports. + * Normalizes common variant attribute names and typos. * * @param {string} name * @returns {string} @@ -144,8 +182,7 @@ export function normalizeAttributeName(name) { } /** - * Extracts or derives a variant attribute value for a child article - * based on its name relative to its parent name, or SKU suffixes. + * Fallback variant value extraction if child has no explicit attribute value. * * @param {string} parentName * @param {string} childName @@ -158,32 +195,15 @@ export function extractVariantValue(parentName, childName, childSku, parentSku) let cName = (childName || '').replace(/[©®™]/g, '').trim(); if (pName && cName) { - // Child name starts with parent name if (cName.toLowerCase().startsWith(pName.toLowerCase())) { let val = cName.slice(pName.length).trim().replace(/^[-–—:/,\s]+/, '').trim(); if (val) return val; } - // Match common leading tokens - const pWords = pName.toLowerCase().split(/\s+/); - const cWords = cName.split(/\s+/); - if (cWords.length > 1 && pWords[0] === cWords[0].toLowerCase()) { - let matchCount = 0; - while (matchCount < pWords.length && matchCount < cWords.length && pWords[matchCount] === cWords[matchCount].toLowerCase()) { - matchCount++; - } - if (matchCount > 0) { - const remaining = cWords.slice(matchCount).join(' ').trim().replace(/^[-–—:/,\s]+/, '').trim(); - if (remaining) return remaining; - } - } } - - // SKU suffix: "42001505-1" with parent "42001505" -> "1" if (parentSku && childSku && childSku.startsWith(parentSku)) { let suffix = childSku.slice(parentSku.length).trim().replace(/^[-_]+/, '').trim(); if (suffix) return suffix; } - if (cName && cName !== pName) return cName; return childSku || 'Standard'; } @@ -215,23 +235,139 @@ export async function executeCompanyImport({ throw new Error('Import files missing: categories.csv and articles.csv are required'); } - const catRaw = fs.readFileSync(catCsvPath, 'utf8'); - const catRecords = parseCsvRecords(catRaw); + const catRecords = parseCsvRecords(fs.readFileSync(catCsvPath, 'utf8')); + const artRecords = parseCsvRecords(fs.readFileSync(artCsvPath, 'utf8')); - const artRaw = fs.readFileSync(artCsvPath, 'utf8'); - const artRecords = parseCsvRecords(artRaw); + log.info(`Read ${catRecords.length} category rows and ${artRecords.length} article rows`); - console.log(`[Import] Read ${catRecords.length} category rows and ${artRecords.length} article rows`); + // ─── Step 1: Filter excluded categories (Papierkorb / Trash and descendants) ─── + const excludedJtlIds = new Set(['72']); + for (const c of catRecords) { + let name = (c.cName || '').trim(); + if (!name) { + try { + const tr = JSON.parse(c.translations || '[]'); + name = tr[0]?.cName || ''; + } catch {} + } + if (name.toLowerCase().includes('papierkorb') || name.toLowerCase().includes('trash')) { + excludedJtlIds.add(String(c.kKategorie)); + } + } + let addedExcluded = true; + while (addedExcluded) { + addedExcluded = false; + for (const c of catRecords) { + const id = String(c.kKategorie); + const parent = String(c.kOberKategorie); + if (!excludedJtlIds.has(id) && excludedJtlIds.has(parent)) { + excludedJtlIds.add(id); + addedExcluded = true; + } + } + } - // Global counts tracking for 200ms progress stream — nothing else is sent + const includedCategories = catRecords.filter((c) => !excludedJtlIds.has(String(c.kKategorie))); + + // Filter included articles (exclude if all its assigned categories are excluded) + const includedArticles = artRecords.filter((a) => { + const catStr = String(a.categories || '').trim(); + if (!catStr) return true; + let catIds = []; + if (catStr.startsWith('[') || catStr.startsWith('{')) { + try { + const raw = JSON.parse(catStr); + catIds = (Array.isArray(raw) ? raw : [raw]).map((it) => String(it.kKategorie || it)); + } catch { + catIds = catStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean); + } + } else { + catIds = catStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean); + } + if (catIds.length > 0 && catIds.every((cid) => excludedJtlIds.has(cid))) { + return false; + } + return true; + }); + + // ─── Step 2: Pre-count All Images and Receipts for Baseline SSE Message ─── + // Category images + const categoryImageTasks = []; + for (const c of includedCategories) { + const picRel = (c.pictureFiles || '').trim(); + if (picRel) { + for (const rel of picRel.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean)) { + const full = path.resolve(importDir, rel); + if (fs.existsSync(full)) { + categoryImageTasks.push({ catJtlId: String(c.kKategorie), relPath: rel, fullPath: full }); + } + } + } + } + + // Brand images (unique manufacturers, 1 primary logo) + const brandMap = new Map(); // brandName -> array of { relPath, fullPath } + for (const a of includedArticles) { + const brand = cleanValue(a.Hersteller); + if (!brand) continue; + if (!brandMap.has(brand)) { + brandMap.set(brand, []); + const rawPics = String(a.manufacturerPictureFiles || '').trim(); + if (rawPics) { + for (const rel of rawPics.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean)) { + const full = path.resolve(importDir, rel); + if (fs.existsSync(full)) { + brandMap.get(brand).push({ relPath: rel, fullPath: full }); + } + } + } + } + } + const brandImageTasks = []; + for (const [brandName, pics] of brandMap.entries()) { + if (pics.length > 0) { + brandImageTasks.push({ brandName, ...pics[0] }); + } + } + + // Article images + const articleImageTasks = []; + for (const a of includedArticles) { + const sku = cleanValue(a.cArtNr); + if (!sku) continue; + const rawPics = String(a.pictureFiles || '').trim(); + if (rawPics) { + for (const rel of rawPics.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean)) { + const full = path.resolve(importDir, rel); + if (fs.existsSync(full)) { + articleImageTasks.push({ sku, relPath: rel, fullPath: full }); + } + } + } + } + + const totalImagesCount = categoryImageTasks.length + brandImageTasks.length + articleImageTasks.length; + + // Receipts / Manifests + const manCsvPath = path.join(importDir, 'manifests.csv'); + const itemsCsvPath = path.join(importDir, 'manifestitems.csv'); + let manRecords = []; + let itemRecords = []; + if (fs.existsSync(manCsvPath) && fs.existsSync(itemsCsvPath)) { + manRecords = parseCsvRecords(fs.readFileSync(manCsvPath, 'utf8')); + itemRecords = parseCsvRecords(fs.readFileSync(itemsCsvPath, 'utf8')); + } + const totalReceiptsCount = manRecords.length; + + // ─── Step 3: Initialize Progress State and Emit Immediate Baseline ─── const progressCounts = { - categoriestotal: 0, + categoriestotal: includedCategories.length, categoriesdone: 0, - articlestotal: 0, + articlestotal: includedArticles.length, articlesdone: 0, - imagestotal: 0, + imagestotal: totalImagesCount, imagesdone: 0, - receiptstotal: 0, + receiptstotal: totalReceiptsCount, receiptsdone: 0, }; @@ -244,7 +380,7 @@ export async function executeCompanyImport({ } }; - // Buffer skelehashes and emit array every 50ms (or on flush) + // Buffer skelehashes and emit array every 50ms let skeleBuffer = []; let lastSkeleEmit = Date.now(); const emitSkeleHash = (base62, force = false) => { @@ -266,42 +402,10 @@ export async function executeCompanyImport({ } }; - // Done-tracking: an image is done when it is stored in the database, an - // article is done when all its images are done, a category is done when - // all its articles are done. - const imagesBySku = new Map(); // sku -> { total, done, paths } - const articlesByCatId = new Map(); // dashCategoryId -> Set - const categoriesBySku = new Map(); // sku -> Set - const doneSku = new Set(); - const doneCatIds = new Set(); + // First SSE message emitted immediately with the exact baseline counts + emitProgress(true); - const markArticleDone = (sku) => { - if (doneSku.has(sku)) return; - doneSku.add(sku); - progressCounts.articlesdone++; - const cats = categoriesBySku.get(sku); - if (!cats) return; - for (const catId of cats) { - if (doneCatIds.has(catId)) continue; - const artSet = articlesByCatId.get(catId); - if (artSet && Array.from(artSet).every((s) => doneSku.has(s))) { - doneCatIds.add(catId); - progressCounts.categoriesdone++; - } - } - }; - - const markImageDone = (sku) => { - progressCounts.imagesdone++; - if (!sku) return; - const rec = imagesBySku.get(sku); - if (!rec) return; - rec.done++; - if (rec.done >= rec.total) markArticleDone(sku); - }; - - // ─── Phase 1: Category Mapping & Creation ──────────────────────────────── - // 1) Find existing category tree & company root category + // ─── Phase 1: Categories Import ────────────────────────────────────────── const treeResult = await store.category.getTreeForCompany(googleAuthId); const existingTree = treeResult.tree || []; const rootCategory = existingTree.find((c) => c.parentId == null) || existingTree[0]; @@ -310,67 +414,43 @@ export async function executeCompanyImport({ } const rootCategoryId = rootCategory.id; - // Helper map of existing categories by parentId and lowercase name const existingMap = new Map(); for (const c of existingTree) { const key = `${c.parentId ?? 'root'}:${String(c.name || '').trim().toLowerCase()}`; existingMap.set(key, c.id); } - // Identify excluded categories (Papierkorb / Trash and its descendants) - const excludedJtlIds = new Set(['72']); - for (const c of catRecords) { - let name = ''; - try { - const tr = JSON.parse(c.translations || '[]'); - name = tr[0]?.cName || ''; - } catch {} - if (name.toLowerCase().includes('papierkorb') || name.toLowerCase().includes('trash')) { - excludedJtlIds.add(String(c.kKategorie)); - } - } - // Also exclude any children of excluded categories - let addedExcluded = true; - while (addedExcluded) { - addedExcluded = false; - for (const c of catRecords) { - const id = String(c.kKategorie); - const parent = String(c.kOberKategorie); - if (!excludedJtlIds.has(id) && excludedJtlIds.has(parent)) { - excludedJtlIds.add(id); - addedExcluded = true; - } - } - } - - console.log(`[Import] Excluded categories (trash/papierkorb):`, Array.from(excludedJtlIds)); - - // Filter included categories - const includedCategories = catRecords.filter((c) => !excludedJtlIds.has(String(c.kKategorie))); - progressCounts.categoriestotal = includedCategories.length; - emitProgress(true); - - // Sort categories so parents are created before children - // JTL kOberKategorie === '0' are top-level const jtlCatIdToDashCatId = new Map(); + jtlCatIdToDashCatId.set('0', rootCategoryId); + + // Group category images by JTL category ID + const catImagesByJtlId = new Map(); + for (const task of categoryImageTasks) { + if (!catImagesByJtlId.has(task.catJtlId)) { + catImagesByJtlId.set(task.catJtlId, []); + } + catImagesByJtlId.get(task.catJtlId).push(task); + } + const pendingCats = [...includedCategories]; let createdCatCount = 0; let catPicCount = 0; - let safety = 0; + while (pendingCats.length > 0 && safety < 10) { safety++; const remaining = []; - for (let i = 0; i < pendingCats.length; i++) { - const c = pendingCats[i]; + for (const c of pendingCats) { const jtlId = String(c.kKategorie); - const jtlParent = String(c.kOberKategorie); + const jtlParent = String(c.kOberKategorie || '0'); - let name = ''; - try { - const tr = JSON.parse(c.translations || '[]'); - name = tr[0]?.cName || ''; - } catch {} + let name = cleanValue(c.cName); + if (!name) { + try { + const tr = JSON.parse(c.translations || '[]'); + name = tr[0]?.cName || ''; + } catch {} + } if (!name) name = `Kategorie ${jtlId}`; let dashParentId; @@ -379,13 +459,11 @@ export async function executeCompanyImport({ } else if (jtlCatIdToDashCatId.has(jtlParent)) { dashParentId = jtlCatIdToDashCatId.get(jtlParent); } else { - // Parent not created yet in this pass remaining.push(c); continue; } - // Check if category already exists under dashParentId - const key = `${dashParentId}:${name.trim().toLowerCase()}`; + const key = `${dashParentId}:${name.toLowerCase()}`; let dashCatId = existingMap.get(key); if (!dashCatId) { @@ -393,136 +471,66 @@ export async function executeCompanyImport({ const created = await store.category.createUnderParent({ googleAuthId, parentId: dashParentId, - name: name.trim(), + name, }); dashCatId = created.id; existingMap.set(key, dashCatId); createdCatCount++; } catch (err) { - console.warn(`[Import] Failed to create category "${name}":`, err.message); + log.warn(`Failed to create category "${name}":`, err.message); } } if (dashCatId) { jtlCatIdToDashCatId.set(jtlId, dashCatId); - // Check for category picture - const picRel = (c.pictureFiles || '').trim(); - if (picRel) { - const picPath = path.join(importDir, picRel); - if (fs.existsSync(picPath)) { - progressCounts.imagestotal++; - try { - const imgBuf = fs.readFileSync(picPath); - await store.category.addImage(googleAuthId, dashCatId, imgBuf); - catPicCount++; + // Add category images + const imgs = catImagesByJtlId.get(jtlId) || []; + for (const imgTask of imgs) { + try { + const imgBuf = fs.readFileSync(imgTask.fullPath); + await store.category.addImage(googleAuthId, dashCatId, imgBuf); + catPicCount++; - // Hash is only emitted after the image is stored - const skele = await computeSkeleHash(imgBuf); - emitSkeleHash(skele?.base62 || null); - markImageDone(null); - } catch (err) { - console.warn(`[Import] Category image error (${picRel}):`, err.message); - } + const skele = await computeSkeleHash(imgBuf); + emitSkeleHash(skele?.base62 || null); + } catch (err) { + log.warn(`Category image error (${imgTask.relPath}):`, err.message); } + progressCounts.imagesdone++; } } + progressCounts.categoriesdone++; emitProgress(); } pendingCats.length = 0; pendingCats.push(...remaining); } + flushSkeleHashes(); emitProgress(true); + log.info(`Categories resolved: ${jtlCatIdToDashCatId.size}/${includedCategories.length}, created: ${createdCatCount}, images: ${catPicCount}`); - console.log(`[Import] Categories resolved: ${jtlCatIdToDashCatId.size}/${includedCategories.length}, created new: ${createdCatCount}, images: ${catPicCount}`); - - // Fallback: Haupt category ID for unmapped items const hauptDashId = jtlCatIdToDashCatId.get('1') || rootCategoryId; - const includedArticles = artRecords.filter((a) => { - let cats = []; - try { - const rawCats = JSON.parse(a.categories || '[]'); - cats = rawCats.map((it) => String(it.kKategorie)); - } catch {} - // If all assigned categories are excluded, exclude article - if (cats.length > 0 && cats.every((cid) => excludedJtlIds.has(cid))) { - return false; - } - return true; - }); - - // Precompute per-article image counts and article/category relations for - // done-tracking (article done = all its images stored, category done = - // all its articles done) - let articleCount = 0; - for (const a of includedArticles) { - const sku = String(a.cArtNr || '').trim(); - if (!sku) continue; - articleCount++; - - const rawPics = (a.pictureFiles || '').trim(); - if (rawPics) { - const lines = rawPics.replace(/\r/g, '\n').split('\n').map((l) => l.trim()).filter(Boolean); - if (lines.length > 0) { - imagesBySku.set(sku, { total: lines.length, done: 0, paths: lines }); - progressCounts.imagestotal += lines.length; - } - } - - let dashCatIds = []; - try { - const rawCats = JSON.parse(a.categories || '[]'); - for (const it of rawCats) { - const jtlCId = String(it.kKategorie); - if (jtlCatIdToDashCatId.has(jtlCId)) { - dashCatIds.push(jtlCatIdToDashCatId.get(jtlCId)); - } - } - } catch {} - for (const catId of dashCatIds) { - if (!articlesByCatId.has(catId)) articlesByCatId.set(catId, new Set()); - articlesByCatId.get(catId).add(sku); - if (!categoriesBySku.has(sku)) categoriesBySku.set(sku, new Set()); - categoriesBySku.get(sku).add(catId); - } - } - progressCounts.articlestotal = articleCount; - emitProgress(true); - - // ─── Phase 2: Brand (Hersteller) Import ────────────────────────────────── - const brandMap = new Map(); // brandName -> Set of picture relative paths - for (const a of includedArticles) { - const hersteller = String(a.Hersteller || '').trim(); - if (!hersteller) continue; - if (!brandMap.has(hersteller)) { - brandMap.set(hersteller, new Set()); - } - const rawPics = String(a.manufacturerPictureFiles || '').trim(); - if (rawPics) { - for (const p of rawPics.split(/[;,]/).map((s) => s.trim()).filter(Boolean)) { - brandMap.get(hersteller).add(p); - } - } - } - + // ─── Phase 2: Brands (Hersteller) Import ────────────────────────────────── const brandNameToId = new Map(); let createdBrandCount = 0; let brandPicCount = 0; - const brandEntries = Array.from(brandMap.entries()); - log.info(`Processing ${brandEntries.length} brands (Hersteller)...`); - // Fetch existing brands to accurately track new vs existing const { brands: existingBrandRows } = await store.brand.list(googleAuthId, { offset: 0, limit: 100 }); const existingBrandNames = new Set( existingBrandRows.map((b) => (b.name || '').trim().toLowerCase()).filter(Boolean) ); - for (let i = 0; i < brandEntries.length; i++) { - const [brandName, picSet] = brandEntries[i]; - const lowerName = brandName.trim().toLowerCase(); + const brandTaskMap = new Map(); + for (const task of brandImageTasks) { + brandTaskMap.set(task.brandName.toLowerCase(), task); + } + + for (const [brandName] of brandMap.entries()) { + const lowerName = brandName.toLowerCase(); const isNew = !existingBrandNames.has(lowerName); try { @@ -533,27 +541,21 @@ export async function executeCompanyImport({ existingBrandNames.add(lowerName); } - // If brand doesn't have an image yet, import from manufacturerPictureFiles - if ((!brand.pics || brand.pics.length === 0) && picSet.size > 0) { - for (const picRel of picSet) { - const picPath = path.resolve(importDir, picRel); - if (fs.existsSync(picPath)) { - progressCounts.imagestotal++; - try { - const imgBuf = fs.readFileSync(picPath); - await store.brand.addImage(googleAuthId, brand.id, imgBuf); - brandPicCount++; + const imgTask = brandTaskMap.get(lowerName); + if (imgTask) { + if (!brand.pics || brand.pics.length === 0) { + try { + const imgBuf = fs.readFileSync(imgTask.fullPath); + await store.brand.addImage(googleAuthId, brand.id, imgBuf); + brandPicCount++; - // Hash is only emitted after the image is stored - const skele = await computeSkeleHash(imgBuf); - emitSkeleHash(skele?.base62 || null); - markImageDone(null); - break; // 1 primary logo per brand - } catch (err) { - log.warn(`Brand image error for "${brandName}" (${picRel}):`, err.message); - } + const skele = await computeSkeleHash(imgBuf); + emitSkeleHash(skele?.base62 || null); + } catch (err) { + log.warn(`Brand image error for "${brandName}":`, err.message); } } + progressCounts.imagesdone++; } } catch (err) { log.warn(`Failed to upsert brand "${brandName}":`, err.message); @@ -561,68 +563,70 @@ export async function executeCompanyImport({ emitProgress(); } + flushSkeleHashes(); emitProgress(true); + log.info(`Brands resolved: ${brandNameToId.size}, created: ${createdBrandCount}, images: ${brandPicCount}`); - log.info(`Brands resolved: ${brandNameToId.size}/${brandEntries.length}, brand images: ${brandPicCount}`); - - // ─── Phase 3: Article Import Pass 1 (Create or Update) ─────────────────── - const totalArticles = includedArticles.length; - log.info(`Articles to process: ${totalArticles}`); - emitProgress(true); - + // ─── Phase 3: Articles Import (Create or Update) ───────────────────────── + const jtlArtIdToSku = new Map(); let articlesImported = 0; let articlesUpdated = 0; - const jtlArtIdToSku = new Map(); - for (let i = 0; i < totalArticles; i++) { - const a = includedArticles[i]; + for (const a of includedArticles) { const jtlArtId = String(a.kArtikel); - const sku = String(a.cArtNr || '').trim(); - if (!sku) continue; + const sku = cleanValue(a.cArtNr); + if (!sku) { + progressCounts.articlesdone++; + continue; + } jtlArtIdToSku.set(jtlArtId, sku); - let name = ''; - let desc = ''; - let seo = ''; + let name = cleanValue(a.cName); + let desc = cleanValue(a.descriptions); - try { - const descriptions = JSON.parse(a.descriptions || '[]'); - const descItem = Array.isArray(descriptions) ? descriptions[0] : descriptions; - if (descItem) { - name = String(descItem.cName || '').trim(); - desc = String(descItem.cBeschreibung || '').trim(); - seo = String(descItem.cKurzBeschreibung || '').trim(); - } - } catch {} + if (!name && desc) { + try { + const parsed = JSON.parse(desc); + const item = Array.isArray(parsed) ? parsed[0] : parsed; + if (item?.cName) name = String(item.cName).trim(); + if (item?.cBeschreibung) desc = String(item.cBeschreibung).trim(); + } catch {} + } if (!name) name = `Artikel ${sku}`; // Category mapping const assignedDashCatIds = []; - try { - const rawCats = JSON.parse(a.categories || '[]'); - for (const it of rawCats) { - const jtlCId = String(it.kKategorie); + const rawCatStr = String(a.categories || '').trim(); + if (rawCatStr) { + let parsedIds = []; + if (rawCatStr.startsWith('[') || rawCatStr.startsWith('{')) { + try { + const raw = JSON.parse(rawCatStr); + parsedIds = (Array.isArray(raw) ? raw : [raw]).map((it) => String(it.kKategorie || it)); + } catch { + parsedIds = rawCatStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean); + } + } else { + parsedIds = rawCatStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean); + } + for (const jtlCId of parsedIds) { if (jtlCatIdToDashCatId.has(jtlCId)) { assignedDashCatIds.push(jtlCatIdToDashCatId.get(jtlCId)); } } - } catch {} + } const primaryCatId = assignedDashCatIds[0] || hauptDashId; - // Stock tracking const trackStock = a.cLagerAktiv === 'Y' ? 1 : 0; - - // Price & Tax - const netPrice = parseFloat(String(a.fVKNetto || '0').replace(',', '.')) || 0; - // kSteuerklasse '3' is reduced tax (7%), '1' is standard tax (19%) + const netPrice = parseGermanFloat(a.fVKNetto, 0); const taxRate = a.kSteuerklasse === '3' ? 7 : 19; + const barcode = cleanValue(a.cBarcode); - // Barcode / EAN - const barcode = String(a.cBarcode || '').trim(); + const hersteller = cleanValue(a.Hersteller); + const brandId = hersteller ? (brandNameToId.get(hersteller.toLowerCase()) || null) : null; - // Check if article already exists let existingArticle = null; try { const res = await store.article.getBySku(googleAuthId, sku); @@ -633,25 +637,20 @@ export async function executeCompanyImport({ try { if (existingArticle) { - // Update existing article await store.article.updateBySku(googleAuthId, sku, { - name: name.trim(), + name, track_stock: trackStock, desc: desc || undefined, - seo: seo || undefined, }); - // Update prices await store.article.setPricesBySku(googleAuthId, sku, [ { net: netPrice, tax: taxRate }, ]); - // Update category assignment if (assignedDashCatIds.length > 0) { await store.article.setCategoryIdsBySku(googleAuthId, sku, assignedDashCatIds); } - // Update barcode if provided if (barcode) { try { const currentEans = existingArticle.eans || []; @@ -659,13 +658,10 @@ export async function executeCompanyImport({ await store.article.setEansBySku(googleAuthId, sku, [...currentEans, barcode]); } } catch (eanErr) { - console.warn(`[Import] Could not set EAN for ${sku}:`, eanErr.message); + log.warn(`Could not set EAN for ${sku}:`, eanErr.message); } } - // Update brand if provided - const hersteller = String(a.Hersteller || '').trim(); - const brandId = hersteller ? (brandNameToId.get(hersteller.toLowerCase()) || null) : null; if (brandId && existingArticle.brand?.id !== brandId) { try { await store.article.setBrandIdBySku(googleAuthId, sku, brandId); @@ -676,39 +672,31 @@ export async function executeCompanyImport({ articlesUpdated++; } else { - // Create new article await store.article.createUnderParent({ googleAuthId, parentId: primaryCatId, - name: name.trim(), + name, sku, track_stock: trackStock, desc: desc || undefined, - seo: seo || undefined, }); - // Set price await store.article.setPricesBySku(googleAuthId, sku, [ { net: netPrice, tax: taxRate }, ]); - // Set additional categories if more than 1 if (assignedDashCatIds.length > 1) { await store.article.setCategoryIdsBySku(googleAuthId, sku, assignedDashCatIds); } - // Set barcode if present if (barcode) { try { await store.article.setEansBySku(googleAuthId, sku, [barcode]); } catch (eanErr) { - console.warn(`[Import] Could not set EAN for ${sku}:`, eanErr.message); + log.warn(`Could not set EAN for ${sku}:`, eanErr.message); } } - // Set brand if provided - const hersteller = String(a.Hersteller || '').trim(); - const brandId = hersteller ? (brandNameToId.get(hersteller.toLowerCase()) || null) : null; if (brandId) { try { await store.article.setBrandIdBySku(googleAuthId, sku, brandId); @@ -720,27 +708,18 @@ export async function executeCompanyImport({ articlesImported++; } } catch (err) { - console.warn(`[Import] Failed to save article ${sku} ("${name}"):`, err.message); + log.warn(`Failed to save article ${sku} ("${name}"):`, err.message); } - // Articles without images are done as soon as their row is written - if (!imagesBySku.has(sku)) markArticleDone(sku); - + progressCounts.articlesdone++; emitProgress(); } emitProgress(true); - console.log(`[Import] Article pass 1 complete. New: ${articlesImported}, Updated: ${articlesUpdated}`); + log.info(`Articles complete: New: ${articlesImported}, Updated: ${articlesUpdated}`); - // ─── Phase 3: Article Import Pass 2 (Variant Links) ────────────────────── - const variantChildren = includedArticles.filter( - (a) => a.kVaterArtikel && a.kVaterArtikel !== '0' && a.kVaterArtikel.trim() !== '' - ); - console.log(`[Import] Linking ${variantChildren.length} child variants to parents...`); - emitProgress(); - - // Fetch existing attribute names for this company - const attributeMap = new Map(); // lowercase name -> attributeId + // ─── Phase 4: Simplified Variant Linking ───────────────────────────────── + const attributeMap = new Map(); try { const attrList = await store.attributename.list(googleAuthId); for (const attr of attrList) { @@ -752,7 +731,6 @@ export async function executeCompanyImport({ log.warn('Could not list attributes:', err.message); } - // Helper to resolve or create an attribute name const resolveAttributeId = async (rawName) => { const normalized = normalizeAttributeName(rawName); const key = normalized.toLowerCase(); @@ -772,12 +750,14 @@ export async function executeCompanyImport({ } }; - // Group children by parent SKU and preserve parent row reference + // Group children by parent SKU const parentChildrenMap = new Map(); - for (const child of variantChildren) { + for (const child of includedArticles) { const parentJtlId = String(child.kVaterArtikel || '').trim(); + if (!parentJtlId || parentJtlId === '0') continue; + const parentSku = jtlArtIdToSku.get(parentJtlId); - const childSku = String(child.cArtNr || '').trim(); + const childSku = cleanValue(child.cArtNr); if (!parentSku || !childSku || parentSku === childSku) continue; if (!parentChildrenMap.has(parentSku)) { @@ -788,77 +768,25 @@ export async function executeCompanyImport({ } let variantsLinked = 0; - let processedCount = 0; - for (const [parentSku, { parentRow, children }] of parentChildrenMap.entries()) { - let parentDetail = null; - try { - const res = await store.article.getBySku(googleAuthId, parentSku); - parentDetail = res.article; - } catch (err) { - log.warn(`[Import] Could not load parent ${parentSku}:`, err.message); - processedCount += children.length; - continue; - } - - // Determine variant attribute name from parent row or children variations - let rawAttrName = null; - try { - const pVars = JSON.parse(parentRow?.variations || '[]'); - rawAttrName = pVars[0]?.cEigenschaftName?.trim() || null; - } catch {} - - if (!rawAttrName) { - for (const child of children) { - try { - const cVars = JSON.parse(child.variations || '[]'); - rawAttrName = cVars[0]?.cEigenschaftName?.trim() || null; - if (rawAttrName) break; - } catch {} - } - } - + const rawAttrName = parentRow ? cleanValue(parentRow.kEigenschaftKombi) : ''; const targetAttrId = await resolveAttributeId(rawAttrName || 'Variante'); - // Ensure parent has is_parent = 1 and parentattributename_id set - let parentAttrId = parentDetail?.parentAttributeNameId; - if (!parentDetail?.isParent || !parentAttrId) { - if (targetAttrId) { - try { - await store.article.toggleParent(googleAuthId, parentSku, { - isParent: true, - attributeNameId: targetAttrId, - }); - parentAttrId = targetAttrId; - } catch (err) { - log.warn(`[Import] Could not toggle parent for ${parentSku}:`, err.message); - } + if (targetAttrId) { + try { + await store.article.toggleParent(googleAuthId, parentSku, { + isParent: true, + attributeNameId: targetAttrId, + }); + } catch (err) { + log.warn(`Could not toggle parent for ${parentSku}:`, err.message); } } - const parentName = parentDetail?.name || ''; const usedValues = new Set(); - for (const child of children) { - processedCount++; - const childSku = String(child.cArtNr || '').trim(); - let childName = ''; - try { - const desc = JSON.parse(child.descriptions || '[]'); - childName = (Array.isArray(desc) ? desc[0]?.cName : desc?.cName) || ''; - } catch {} - - // Determine attribute value from child variations, falling back to name derivation - let val = null; - try { - const cVars = JSON.parse(child.variations || '[]'); - val = cVars[0]?.selectedValues?.[0]?.cEigenschaftWertName?.trim() || - cVars[0]?.variationValues?.[0]?.cEigenschaftWertName?.trim() || null; - } catch {} - - if (!val) { - val = extractVariantValue(parentName, childName, childSku, parentSku); - } + const childSku = cleanValue(child.cArtNr); + let val = cleanValue(child.kEigenschaftKombi) || 'Standard'; if (usedValues.has(val.toLowerCase())) { val = `${val} (${childSku})`; @@ -866,7 +794,7 @@ export async function executeCompanyImport({ usedValues.add(val.toLowerCase()); try { - if (parentAttrId) { + if (targetAttrId) { await store.article.attachChild(googleAuthId, parentSku, { sku: childSku, value: val, @@ -874,85 +802,42 @@ export async function executeCompanyImport({ }); variantsLinked++; } else { - // Fallback to setParentBySku if toggleParent failed await store.article.setParentBySku(googleAuthId, childSku, parentSku); variantsLinked++; } } catch (err) { - console.warn(`[Import] Failed to attach child ${childSku} to parent ${parentSku}:`, err.message); + log.warn(`Failed to attach child ${childSku} to parent ${parentSku}:`, err.message); } - - emitProgress(); } } - console.log(`[Import] Linked ${variantsLinked}/${variantChildren.length} variants`); - - // ─── Phase 4: Article Pictures Import ──────────────────────────────────── - const imageTasks = []; - for (const [sku, rec] of imagesBySku.entries()) { - for (const relPath of rec.paths) { - imageTasks.push({ sku, relPath }); - } - } - - console.log(`[Import] Importing ${imageTasks.length} pictures for ${imagesBySku.size} articles...`); + log.info(`Linked ${variantsLinked} variants across ${parentChildrenMap.size} parents`); + // ─── Phase 5: Article Pictures Import ──────────────────────────────────── let totalImagesImported = 0; - for (let i = 0; i < imageTasks.length; i++) { - const { sku, relPath } = imageTasks[i]; - const imgPath = path.join(importDir, relPath); + for (const task of articleImageTasks) { + try { + const imgBuf = fs.readFileSync(task.fullPath); + await store.article.addImage(googleAuthId, task.sku, imgBuf); + totalImagesImported++; - if (fs.existsSync(imgPath)) { - try { - const imgBuf = fs.readFileSync(imgPath); - await store.article.addImage(googleAuthId, sku, imgBuf); - totalImagesImported++; - - // Hash is only emitted after the image is stored in the database - const skele = await computeSkeleHash(imgBuf); - emitSkeleHash(skele?.base62 || null); - markImageDone(sku); - } catch (err) { - // If already attached or format issue, log and continue - console.warn(`[Import] Image add failed for SKU ${sku} (${relPath}):`, err.message); - } + const skele = await computeSkeleHash(imgBuf); + emitSkeleHash(skele?.base62 || null); + } catch (err) { + log.warn(`Article image add failed for ${task.sku} (${task.relPath}):`, err.message); } - + progressCounts.imagesdone++; emitProgress(); } + flushSkeleHashes(); - - // Reconcile done-counters so articles/categories reach their totals even - // when individual images failed - for (const sku of imagesBySku.keys()) markArticleDone(sku); - for (const catId of jtlCatIdToDashCatId.values()) { - if (doneCatIds.has(catId)) continue; - const artSet = articlesByCatId.get(catId); - if (!artSet || artSet.size === 0 || Array.from(artSet).every((s) => doneSku.has(s))) { - doneCatIds.add(catId); - progressCounts.categoriesdone++; - } - } emitProgress(true); + log.info(`Article pictures complete: ${totalImagesImported}/${articleImageTasks.length}`); - console.log(`[Import] Completed article pictures: ${totalImagesImported} images processed`); - - // ─── Phase 5: Orders (Manifests) Import ────────────────────────────────── - const manCsvPath = path.join(importDir, 'manifests.csv'); - const itemsCsvPath = path.join(importDir, 'manifestitems.csv'); + // ─── Phase 6: Orders / Receipts (Manifests) Import ─────────────────────── let ordersImported = 0; let ordersSkipped = 0; - let totalOrders = 0; - if (fs.existsSync(manCsvPath) && fs.existsSync(itemsCsvPath)) { - const manRecords = parseCsvRecords(fs.readFileSync(manCsvPath, 'utf8')); - const itemRecords = parseCsvRecords(fs.readFileSync(itemsCsvPath, 'utf8')); - totalOrders = manRecords.length; - console.log(`[Import] Manifests found: ${totalOrders} orders, ${itemRecords.length} items`); - progressCounts.receiptstotal = totalOrders; - emitProgress(true); - - // Group line items by kAuftrag + if (manRecords.length > 0 && itemRecords.length > 0) { const itemsByAuftrag = new Map(); for (const it of itemRecords) { const kAuftrag = String(it.kAuftrag || '').trim(); @@ -963,20 +848,21 @@ export async function executeCompanyImport({ itemsByAuftrag.get(kAuftrag).push(it); } - // Pre-fetch existing orders to make import idempotent - const existingOrders = await store.order.list(googleAuthId); + const existingOrdersResult = await store.order.list(googleAuthId); + const existingOrderList = Array.isArray(existingOrdersResult) + ? existingOrdersResult + : (existingOrdersResult?.orders || []); const existingOrderNumbers = new Set( - existingOrders.map((o) => (o.number || '').trim()).filter(Boolean) + existingOrderList.map((o) => (o.number || '').trim()).filter(Boolean) ); - // Pre-fetch customer groups and existing entities to resolve customer let defaultCustomerGroupId = null; try { const groups = await store.customergroup.list(googleAuthId); defaultCustomerGroupId = groups[0]?.id || null; } catch {} - const entityMap = new Map(); // lowercase name -> entityId + const entityMap = new Map(); try { const entities = await store.entity.list(googleAuthId); for (const ent of entities) { @@ -1001,20 +887,14 @@ export async function executeCompanyImport({ entityMap.set(key, ent.id); return ent.id; } catch (err) { - log.warn(`[Import] Could not create entity "${cleanName}":`, err.message); + log.warn(`Could not create entity "${cleanName}":`, err.message); return null; } }; - for (let i = 0; i < totalOrders; i++) { - const m = manRecords[i]; - const orderNumber = String(m.cAuftragsNr || '').trim(); - if (!orderNumber) { - progressCounts.receiptsdone++; - continue; - } - - if (existingOrderNumbers.has(orderNumber)) { + for (const m of manRecords) { + const orderNumber = cleanValue(m.cAuftragsNr); + if (!orderNumber || existingOrderNumbers.has(orderNumber)) { ordersSkipped++; progressCounts.receiptsdone++; emitProgress(); @@ -1030,13 +910,14 @@ export async function executeCompanyImport({ } const lines = rawItems.map((it) => { - const sku = (it.cArtNr || '').trim() || null; - const name = (it.cName || it.cNameStandard || '').trim() || (sku ? `Artikel ${sku}` : 'Artikel'); - const amount = parseFloat(String(it.fAnzahl || '1').replace(',', '.')) || 1; - const net = parseFloat(String(it.fVKNetto || '0').replace(',', '.')) || 0; - const taxrate = parseFloat(String(it.fMwSt || '19').replace(',', '.')) || 0; - const rebate = parseFloat(String(it.fRabatt || '0').replace(',', '.')) || 0; - const comment = (it.cHinweis || '').trim() || null; + const sku = cleanValue(it.cArtNr) || null; + const rawName = cleanValue(it.cName || it.cNameStandard); + const name = rawName || (sku ? `Artikel ${sku}` : 'Artikel'); + const amount = parseGermanFloat(it.fAnzahl, 1); + const net = parseGermanFloat(it.fVKNetto, 0); + const taxrate = parseGermanFloat(it.fMwSt, 19); + const rebate = parseGermanFloat(it.fRabatt, 0); + const comment = cleanValue(it.cHinweis) || null; return { sku, @@ -1049,18 +930,18 @@ export async function executeCompanyImport({ }; }); - const custName = (m.cName || '').trim() || 'Laufkunde'; - const custFirstName = (m.cVorname || '').trim(); + const custName = cleanValue(m.cName) || 'Laufkunde'; + const custFirstName = cleanValue(m.cVorname); const entityId = await resolveEntityId(custName, custFirstName); - const dateStr = (m.dErstellt || '').trim(); + const dateStr = cleanValue(m.dErstellt); const parsedDate = dateStr ? new Date(dateStr) : new Date(); const validDate = !Number.isNaN(parsedDate.getTime()) ? parsedDate.toISOString() : new Date().toISOString(); try { await store.order.create(googleAuthId, { number: orderNumber, - externalNumber: (m.cExterneAuftragsnummer || '').trim() || null, + externalNumber: cleanValue(m.cExterneAuftragsnummer) || null, date: validDate, createdAt: validDate, entityId: entityId || undefined, @@ -1069,17 +950,20 @@ export async function executeCompanyImport({ existingOrderNumbers.add(orderNumber); ordersImported++; } catch (err) { - console.warn(`[Import] Failed to create order ${orderNumber}:`, err.message); + log.warn(`Failed to create order ${orderNumber}:`, err.message); } progressCounts.receiptsdone++; emitProgress(); } - emitProgress(true); - console.log(`[Import] Orders import complete: ${ordersImported} created, ${ordersSkipped} skipped`); + emitProgress(true); + log.info(`Orders complete: ${ordersImported} created, ${ordersSkipped} skipped`); } + flushSkeleHashes(); + emitProgress(true); + return { ok: true, message: 'Import erfolgreich abgeschlossen.', @@ -1090,12 +974,12 @@ export async function executeCompanyImport({ brands: brandNameToId.size, brandsCreated: createdBrandCount, brandPictures: brandPicCount, - articlesTotal: totalArticles, + articlesTotal: includedArticles.length, articlesImported, articlesUpdated, variantsLinked, articlePictures: totalImagesImported, - ordersTotal: totalOrders, + ordersTotal: totalReceiptsCount, ordersImported, ordersSkipped, },