diff --git a/src/services/companyImport.js b/src/services/companyImport.js index 2bf8dc5..c955d94 100644 --- a/src/services/companyImport.js +++ b/src/services/companyImport.js @@ -1,212 +1,39 @@ +// MARK: - Imports +// #region Imports import fs from 'fs'; import path from 'path'; -import sharp from 'sharp'; -import { encodeRGBA } from 'skelehash-wasm'; import { getStore } from '../db/store.js'; import { createLogger } from '../util/logger.js'; +import { cleanValue, parseGermanFloat, parseCsvRecords } from './companyImport/csvParser.js'; +import { computeSkeleHash } from './companyImport/imageHasher.js'; +import { normalizeAttributeName, extractVariantValue } from './companyImport/variantHelpers.js'; +import { createProgressTracker } from './companyImport/progressTracker.js'; +import { filterCatalogRecords, prepareImportTasks } from './companyImport/catalogPreprocessor.js'; +import { importCategories } from './companyImport/categoryImporter.js'; +import { importBrands } from './companyImport/brandImporter.js'; +import { importArticles } from './companyImport/articleImporter.js'; +import { importVariants } from './companyImport/variantImporter.js'; +import { importArticleImages } from './companyImport/articleImageImporter.js'; +import { importOrders } from './companyImport/orderImporter.js'; +// #endregion + +// MARK: - Re-exports +// #region Re-exports +export { + cleanValue, + parseGermanFloat, + parseCsvRecords, + computeSkeleHash, + normalizeAttributeName, + extractVariantValue, +}; +// #endregion + const log = createLogger('import'); -/** - * Computes SkeleHash base62 hash for an image buffer. - * - * @param {Buffer} buffer - * @returns {Promise<{base62: string, width: number, height: number, hashLength: number}|null>} - */ -async function computeSkeleHash(buffer) { - try { - const { data, info } = await sharp(buffer) - .rotate() - .resize({ width: 200, height: 200, fit: 'inside' }) - .ensureAlpha() - .raw() - .toBuffer({ resolveWithObject: true }); - - const result = await encodeRGBA(info.width, info.height, data); - return { - base62: result.base62, - width: info.width, - height: info.height, - hashLength: result.hash?.length ?? 0, - }; - } catch (err) { - log.warn('Failed to compute skelehash for image:', err.message); - return null; - } -} - -/** - * Cleans formula prefixes (="..."), quotes, and whitespace from CSV values. - * - * @param {string|any} str - * @returns {string} - */ -export function cleanValue(str) { - if (str == null) return ''; - let val = String(str).trim(); - if (val.startsWith('=')) { - val = val.slice(1).trim(); - } - if (val.startsWith('"') && val.endsWith('"')) { - val = val.slice(1, -1).trim(); - } - return val; -} - -/** - * Parses German decimal format numbers (e.g. "8,36" -> 8.36). - * - * @param {string|number} str - * @param {number} [fallback=0] - * @returns {number} - */ -export function parseGermanFloat(str, fallback = 0) { - const clean = cleanValue(str).replace(',', '.'); - const num = parseFloat(clean); - return Number.isNaN(num) ? fallback : num; -} - -/** - * Parses CSV text into an array of objects keyed by header. - * Supports auto-detecting semicolon (;) vs comma (,) delimiters, - * quoted fields with embedded newlines/quotes, and formula prefixes. - * - * @param {string} text - Raw CSV text - * @returns {Array>} - */ -export function parseCsvRecords(text) { - const clean = text.replace(/^\uFEFF/, ''); - const firstLine = clean.split(/[\r\n]/)[0] || ''; - const semiCount = (firstLine.match(/;/g) || []).length; - const commaCount = (firstLine.match(/,/g) || []).length; - const delimiter = semiCount >= commaCount ? ';' : ','; - - const rows = []; - let field = ''; - let row = []; - let inQuotes = false; - let i = 0; - const len = clean.length; - - while (i < len) { - const c = clean[i]; - if (inQuotes) { - if (c === '"') { - if (i + 1 < len && clean[i + 1] === '"') { - field += '"'; - i += 2; - continue; - } else { - inQuotes = false; - i++; - continue; - } - } else { - field += c; - i++; - continue; - } - } else { - if (c === '"') { - inQuotes = true; - i++; - continue; - } else if (c === delimiter) { - row.push(cleanValue(field)); - field = ''; - i++; - continue; - } else if (c === '\r') { - if (i + 1 < len && clean[i + 1] === '\n') { - i++; - } - row.push(cleanValue(field)); - rows.push(row); - field = ''; - row = []; - i++; - continue; - } else if (c === '\n') { - row.push(cleanValue(field)); - rows.push(row); - field = ''; - row = []; - i++; - continue; - } else { - field += c; - i++; - continue; - } - } - } - if (field !== '' || row.length > 0) { - row.push(cleanValue(field)); - rows.push(row); - } - if (rows.length === 0) return []; - const header = rows[0].map((h) => cleanValue(h)); - const records = []; - for (let r = 1; r < rows.length; r++) { - const curr = rows[r]; - if (curr.length === 1 && curr[0] === '') continue; - const obj = {}; - for (let col = 0; col < header.length; col++) { - obj[header[col]] = curr[col] ?? ''; - } - records.push(obj); - } - return records; -} - -/** - * Normalizes common variant attribute names and typos. - * - * @param {string} name - * @returns {string} - */ -export function normalizeAttributeName(name) { - const clean = (name || '').trim(); - if (!clean) return 'Variante'; - const lower = clean.toLowerCase(); - if (lower === 'menge' || lower === 'mange' || lower === 'meneg') return 'Menge'; - if (lower === 'farbe') return 'Farbe'; - if (lower === 'größe' || lower === 'grosse') return 'Größe'; - if (lower === 'sorte') return 'Sorte'; - if (lower === 'geschmack') return 'Geschmack'; - if (lower === 'typ') return 'Typ'; - if (lower === 'stück' || lower === 'stueck') return 'Stück'; - if (lower === 'ausführung' || lower === 'ausfuehrung') return 'Ausführung'; - if (lower === 'motiv') return 'Motiv'; - return clean.charAt(0).toUpperCase() + clean.slice(1); -} - -/** - * Fallback variant value extraction if child has no explicit attribute value. - * - * @param {string} parentName - * @param {string} childName - * @param {string} childSku - * @param {string} parentSku - * @returns {string} - */ -export function extractVariantValue(parentName, childName, childSku, parentSku) { - let pName = (parentName || '').replace(/[©®™]/g, '').trim(); - let cName = (childName || '').replace(/[©®™]/g, '').trim(); - - if (pName && cName) { - if (cName.toLowerCase().startsWith(pName.toLowerCase())) { - let val = cName.slice(pName.length).trim().replace(/^[-–—:/,\s]+/, '').trim(); - if (val) return val; - } - } - if (parentSku && childSku && childSku.startsWith(parentSku)) { - let suffix = childSku.slice(parentSku.length).trim().replace(/^[-_]+/, '').trim(); - if (suffix) return suffix; - } - if (cName && cName !== pName) return cName; - return childSku || 'Standard'; -} +// MARK: - Company Import Orchestrator +// #region Company Import Orchestrator /** * Executes full catalog import from preserved files in importDir. @@ -215,9 +42,9 @@ export function extractVariantValue(parentName, childName, childSku, parentSku) * @param {number} opts.googleAuthId * @param {number} opts.companyId * @param {string} opts.importDir - * @param {function} [opts.onProgress] - (counts) => void; counts = { categoriestotal, categoriesdone, articlestotal, articlesdone, imagestotal, imagesdone, receiptstotal, receiptsdone }; throttled to 200ms - * @param {function} [opts.onImages] - (skelehashes: string[]) => void; batched every 50ms, a hash is only included after its image was stored in the database - * @returns {Promise} Summary outcome + * @param {function} [opts.onProgress] - (counts) => void; throttled to 200ms + * @param {function} [opts.onImages] - (skelehashes: string[]) => void; batched every 50ms + * @returns {Promise} Summary outcome with statistics */ export async function executeCompanyImport({ googleAuthId, @@ -228,6 +55,8 @@ export async function executeCompanyImport({ }) { const store = await getStore(); + // MARK: - Verify and Read Input Files + // #region Read CSVs const catCsvPath = path.join(importDir, 'categories.csv'); const artCsvPath = path.join(importDir, 'articles.csv'); @@ -237,751 +66,136 @@ export async function executeCompanyImport({ const catRecords = parseCsvRecords(fs.readFileSync(catCsvPath, 'utf8')); const artRecords = parseCsvRecords(fs.readFileSync(artCsvPath, 'utf8')); - log.info(`Read ${catRecords.length} category rows and ${artRecords.length} article rows`); + // #endregion - // ─── Step 1: Filter excluded categories (Papierkorb / Trash and descendants) ─── - const excludedJtlIds = new Set(['72']); - for (const c of catRecords) { - let name = (c.cName || '').trim(); - if (!name) { - try { - const tr = JSON.parse(c.translations || '[]'); - name = tr[0]?.cName || ''; - } catch {} - } - if (name.toLowerCase().includes('papierkorb') || name.toLowerCase().includes('trash')) { - excludedJtlIds.add(String(c.kKategorie)); - } - } - let addedExcluded = true; - while (addedExcluded) { - addedExcluded = false; - for (const c of catRecords) { - const id = String(c.kKategorie); - const parent = String(c.kOberKategorie); - if (!excludedJtlIds.has(id) && excludedJtlIds.has(parent)) { - excludedJtlIds.add(id); - addedExcluded = true; - } - } - } + // MARK: - Filter Records & Prepare Tasks + // #region Preprocessing & Task Discovery + const { includedCategories, includedArticles } = filterCatalogRecords(catRecords, artRecords); - const includedCategories = catRecords.filter((c) => !excludedJtlIds.has(String(c.kKategorie))); + const { + categoryImageTasks, + brandMap, + brandImageTasks, + articleImageTasks, + totalImagesCount, + manRecords, + itemRecords, + totalReceiptsCount, + } = prepareImportTasks({ importDir, includedCategories, includedArticles }); + // #endregion - // Filter included articles (exclude if all its assigned categories are excluded) - const includedArticles = artRecords.filter((a) => { - const catStr = String(a.categories || '').trim(); - if (!catStr) return true; - let catIds = []; - if (catStr.startsWith('[') || catStr.startsWith('{')) { - try { - const raw = JSON.parse(catStr); - catIds = (Array.isArray(raw) ? raw : [raw]).map((it) => String(it.kKategorie || it)); - } catch { - catIds = catStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean); - } - } else { - catIds = catStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean); - } - if (catIds.length > 0 && catIds.every((cid) => excludedJtlIds.has(cid))) { - return false; - } - return true; + // MARK: - Progress Tracker Initialization + // #region Progress Setup + const progressTracker = createProgressTracker({ + initialCounts: { + categoriestotal: includedCategories.length, + categoriesdone: 0, + articlestotal: includedArticles.length, + articlesdone: 0, + imagestotal: totalImagesCount, + imagesdone: 0, + receiptstotal: totalReceiptsCount, + receiptsdone: 0, + }, + onProgress, + onImages, }); - // ─── Step 2: Pre-count All Images and Receipts for Baseline SSE Message ─── - // Category images - const categoryImageTasks = []; - for (const c of includedCategories) { - const picRel = (c.pictureFiles || '').trim(); - if (picRel) { - for (const rel of picRel.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean)) { - const full = path.resolve(importDir, rel); - if (fs.existsSync(full)) { - categoryImageTasks.push({ catJtlId: String(c.kKategorie), relPath: rel, fullPath: full }); - } - } - } - } - - // Brand images (unique manufacturers, 1 primary logo) - const brandMap = new Map(); // brandName -> array of { relPath, fullPath } - for (const a of includedArticles) { - const brand = cleanValue(a.Hersteller); - if (!brand) continue; - if (!brandMap.has(brand)) { - brandMap.set(brand, []); - const rawPics = String(a.manufacturerPictureFiles || '').trim(); - if (rawPics) { - for (const rel of rawPics.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean)) { - const full = path.resolve(importDir, rel); - if (fs.existsSync(full)) { - brandMap.get(brand).push({ relPath: rel, fullPath: full }); - } - } - } - } - } - const brandImageTasks = []; - for (const [brandName, pics] of brandMap.entries()) { - if (pics.length > 0) { - brandImageTasks.push({ brandName, ...pics[0] }); - } - } - - // Article images - const articleImageTasks = []; - for (const a of includedArticles) { - const sku = cleanValue(a.cArtNr); - if (!sku) continue; - const rawPics = String(a.pictureFiles || '').trim(); - if (rawPics) { - for (const rel of rawPics.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean)) { - const full = path.resolve(importDir, rel); - if (fs.existsSync(full)) { - articleImageTasks.push({ sku, relPath: rel, fullPath: full }); - } - } - } - } - - const totalImagesCount = categoryImageTasks.length + brandImageTasks.length + articleImageTasks.length; - - // Receipts / Manifests - const manCsvPath = path.join(importDir, 'manifests.csv'); - const itemsCsvPath = path.join(importDir, 'manifestitems.csv'); - let manRecords = []; - let itemRecords = []; - if (fs.existsSync(manCsvPath) && fs.existsSync(itemsCsvPath)) { - manRecords = parseCsvRecords(fs.readFileSync(manCsvPath, 'utf8')); - itemRecords = parseCsvRecords(fs.readFileSync(itemsCsvPath, 'utf8')); - } - const totalReceiptsCount = manRecords.length; - - // ─── Step 3: Initialize Progress State and Emit Immediate Baseline ─── - const progressCounts = { - categoriestotal: includedCategories.length, - categoriesdone: 0, - articlestotal: includedArticles.length, - articlesdone: 0, - imagestotal: totalImagesCount, - imagesdone: 0, - receiptstotal: totalReceiptsCount, - receiptsdone: 0, - }; - - let lastProgressEmit = 0; - const emitProgress = (force = false) => { - const now = Date.now(); - if (force || now - lastProgressEmit >= 200) { - lastProgressEmit = now; - onProgress({ ...progressCounts }); - } - }; - - // Buffer skelehashes and emit array every 50ms - let skeleBuffer = []; - let lastSkeleEmit = Date.now(); - const emitSkeleHash = (base62, force = false) => { - if (base62) skeleBuffer.push(base62); - const now = Date.now(); - if (skeleBuffer.length > 0 && (force || now - lastSkeleEmit >= 50)) { - const batch = skeleBuffer; - skeleBuffer = []; - lastSkeleEmit = now; - onImages(batch); - } - }; - const flushSkeleHashes = () => { - if (skeleBuffer.length > 0) { - const batch = skeleBuffer; - skeleBuffer = []; - lastSkeleEmit = Date.now(); - onImages(batch); - } - }; - - // First SSE message emitted immediately with the exact baseline counts - emitProgress(true); - - // ─── Phase 1: Categories Import ────────────────────────────────────────── - const treeResult = await store.category.getTreeForCompany(googleAuthId); - const existingTree = treeResult.tree || []; - const rootCategory = existingTree.find((c) => c.parentId == null) || existingTree[0]; - if (!rootCategory) { - throw new Error('Company root category not found'); - } - const rootCategoryId = rootCategory.id; - - const existingMap = new Map(); - for (const c of existingTree) { - const key = `${c.parentId ?? 'root'}:${String(c.name || '').trim().toLowerCase()}`; - existingMap.set(key, c.id); - } - - const jtlCatIdToDashCatId = new Map(); - jtlCatIdToDashCatId.set('0', rootCategoryId); - - // Group category images by JTL category ID - const catImagesByJtlId = new Map(); - for (const task of categoryImageTasks) { - if (!catImagesByJtlId.has(task.catJtlId)) { - catImagesByJtlId.set(task.catJtlId, []); - } - catImagesByJtlId.get(task.catJtlId).push(task); - } - - const pendingCats = [...includedCategories]; - let createdCatCount = 0; - let catPicCount = 0; - let safety = 0; - - while (pendingCats.length > 0 && safety < 10) { - safety++; - const remaining = []; - for (const c of pendingCats) { - const jtlId = String(c.kKategorie); - const jtlParent = String(c.kOberKategorie || '0'); - - let name = cleanValue(c.cName); - if (!name) { - try { - const tr = JSON.parse(c.translations || '[]'); - name = tr[0]?.cName || ''; - } catch {} - } - if (!name) name = `Kategorie ${jtlId}`; - - let dashParentId; - if (jtlParent === '0' || !jtlParent) { - dashParentId = rootCategoryId; - } else if (jtlCatIdToDashCatId.has(jtlParent)) { - dashParentId = jtlCatIdToDashCatId.get(jtlParent); - } else { - remaining.push(c); - continue; - } - - const key = `${dashParentId}:${name.toLowerCase()}`; - let dashCatId = existingMap.get(key); - - if (!dashCatId) { - try { - const created = await store.category.createUnderParent({ - googleAuthId, - parentId: dashParentId, - name, - }); - dashCatId = created.id; - existingMap.set(key, dashCatId); - createdCatCount++; - } catch (err) { - log.warn(`Failed to create category "${name}":`, err.message); - } - } - - if (dashCatId) { - jtlCatIdToDashCatId.set(jtlId, dashCatId); - - // Add category images - const imgs = catImagesByJtlId.get(jtlId) || []; - for (const imgTask of imgs) { - try { - const imgBuf = fs.readFileSync(imgTask.fullPath); - await store.category.addImage(googleAuthId, dashCatId, imgBuf); - catPicCount++; - - const skele = await computeSkeleHash(imgBuf); - emitSkeleHash(skele?.base62 || null); - } catch (err) { - log.warn(`Category image error (${imgTask.relPath}):`, err.message); - } - progressCounts.imagesdone++; - } - } - - progressCounts.categoriesdone++; - emitProgress(); - } - pendingCats.length = 0; - pendingCats.push(...remaining); - } - - flushSkeleHashes(); - emitProgress(true); - log.info(`Categories resolved: ${jtlCatIdToDashCatId.size}/${includedCategories.length}, created: ${createdCatCount}, images: ${catPicCount}`); - - const hauptDashId = jtlCatIdToDashCatId.get('1') || rootCategoryId; - - // ─── Phase 2: Brands (Hersteller) Import ────────────────────────────────── - const brandNameToId = new Map(); - let createdBrandCount = 0; - let brandPicCount = 0; - - const { brands: existingBrandRows } = await store.brand.list(googleAuthId, { offset: 0, limit: 100 }); - const existingBrandNames = new Set( - existingBrandRows.map((b) => (b.name || '').trim().toLowerCase()).filter(Boolean) - ); - - const brandTaskMap = new Map(); - for (const task of brandImageTasks) { - brandTaskMap.set(task.brandName.toLowerCase(), task); - } - - for (const [brandName] of brandMap.entries()) { - const lowerName = brandName.toLowerCase(); - const isNew = !existingBrandNames.has(lowerName); - - try { - const brand = await store.brand.upsert(googleAuthId, { name: brandName }); - brandNameToId.set(lowerName, brand.id); - if (isNew) { - createdBrandCount++; - existingBrandNames.add(lowerName); - } - - const imgTask = brandTaskMap.get(lowerName); - if (imgTask) { - if (!brand.pics || brand.pics.length === 0) { - try { - const imgBuf = fs.readFileSync(imgTask.fullPath); - await store.brand.addImage(googleAuthId, brand.id, imgBuf); - brandPicCount++; - - const skele = await computeSkeleHash(imgBuf); - emitSkeleHash(skele?.base62 || null); - } catch (err) { - log.warn(`Brand image error for "${brandName}":`, err.message); - } - } - progressCounts.imagesdone++; - } - } catch (err) { - log.warn(`Failed to upsert brand "${brandName}":`, err.message); - } - - emitProgress(); - } - - flushSkeleHashes(); - emitProgress(true); - log.info(`Brands resolved: ${brandNameToId.size}, created: ${createdBrandCount}, images: ${brandPicCount}`); - - // ─── Phase 3: Articles Import (Create or Update) ───────────────────────── - const jtlArtIdToSku = new Map(); - let articlesImported = 0; - let articlesUpdated = 0; - - for (const a of includedArticles) { - const jtlArtId = String(a.kArtikel); - const sku = cleanValue(a.cArtNr); - if (!sku) { - progressCounts.articlesdone++; - continue; - } - - jtlArtIdToSku.set(jtlArtId, sku); - - let name = cleanValue(a.cName); - let desc = cleanValue(a.descriptions); - - if (!name && desc) { - try { - const parsed = JSON.parse(desc); - const item = Array.isArray(parsed) ? parsed[0] : parsed; - if (item?.cName) name = String(item.cName).trim(); - if (item?.cBeschreibung) desc = String(item.cBeschreibung).trim(); - } catch {} - } - if (!name) name = `Artikel ${sku}`; - - // Category mapping - const assignedDashCatIds = []; - const rawCatStr = String(a.categories || '').trim(); - if (rawCatStr) { - let parsedIds = []; - if (rawCatStr.startsWith('[') || rawCatStr.startsWith('{')) { - try { - const raw = JSON.parse(rawCatStr); - parsedIds = (Array.isArray(raw) ? raw : [raw]).map((it) => String(it.kKategorie || it)); - } catch { - parsedIds = rawCatStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean); - } - } else { - parsedIds = rawCatStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean); - } - for (const jtlCId of parsedIds) { - if (jtlCatIdToDashCatId.has(jtlCId)) { - assignedDashCatIds.push(jtlCatIdToDashCatId.get(jtlCId)); - } - } - } - const primaryCatId = assignedDashCatIds[0] || hauptDashId; - - const trackStock = a.cLagerAktiv === 'Y' ? 1 : 0; - const netPrice = parseGermanFloat(a.fVKNetto, 0); - const taxRate = a.kSteuerklasse === '3' ? 7 : 19; - const barcode = cleanValue(a.cBarcode); - - const hersteller = cleanValue(a.Hersteller); - const brandId = hersteller ? (brandNameToId.get(hersteller.toLowerCase()) || null) : null; - - let existingArticle = null; - try { - const res = await store.article.getBySku(googleAuthId, sku); - existingArticle = res.article; - } catch { - existingArticle = null; - } - - try { - if (existingArticle) { - await store.article.updateBySku(googleAuthId, sku, { - name, - track_stock: trackStock, - desc: desc || undefined, - }); - - await store.article.setPricesBySku(googleAuthId, sku, [ - { net: netPrice, tax: taxRate }, - ]); - - if (assignedDashCatIds.length > 0) { - await store.article.setCategoryIdsBySku(googleAuthId, sku, assignedDashCatIds); - } - - if (barcode) { - try { - const currentEans = existingArticle.eans || []; - if (!currentEans.includes(barcode)) { - await store.article.setEansBySku(googleAuthId, sku, [...currentEans, barcode]); - } - } catch (eanErr) { - log.warn(`Could not set EAN for ${sku}:`, eanErr.message); - } - } - - if (brandId && existingArticle.brand?.id !== brandId) { - try { - await store.article.setBrandIdBySku(googleAuthId, sku, brandId); - } catch (bErr) { - log.warn(`Could not set brand for ${sku}:`, bErr.message); - } - } - - articlesUpdated++; - } else { - await store.article.createUnderParent({ - googleAuthId, - parentId: primaryCatId, - name, - sku, - track_stock: trackStock, - desc: desc || undefined, - }); - - await store.article.setPricesBySku(googleAuthId, sku, [ - { net: netPrice, tax: taxRate }, - ]); - - if (assignedDashCatIds.length > 1) { - await store.article.setCategoryIdsBySku(googleAuthId, sku, assignedDashCatIds); - } - - if (barcode) { - try { - await store.article.setEansBySku(googleAuthId, sku, [barcode]); - } catch (eanErr) { - log.warn(`Could not set EAN for ${sku}:`, eanErr.message); - } - } - - if (brandId) { - try { - await store.article.setBrandIdBySku(googleAuthId, sku, brandId); - } catch (bErr) { - log.warn(`Could not set brand for ${sku}:`, bErr.message); - } - } - - articlesImported++; - } - } catch (err) { - log.warn(`Failed to save article ${sku} ("${name}"):`, err.message); - } - - progressCounts.articlesdone++; - emitProgress(); - } - - emitProgress(true); - log.info(`Articles complete: New: ${articlesImported}, Updated: ${articlesUpdated}`); - - // ─── Phase 4: Simplified Variant Linking ───────────────────────────────── - const attributeMap = new Map(); - try { - const attrList = await store.attributename.list(googleAuthId); - for (const attr of attrList) { - if (attr.name) { - attributeMap.set(attr.name.trim().toLowerCase(), attr.id); - } - } - } catch (err) { - log.warn('Could not list attributes:', err.message); - } - - const resolveAttributeId = async (rawName) => { - const normalized = normalizeAttributeName(rawName); - const key = normalized.toLowerCase(); - if (attributeMap.has(key)) { - return attributeMap.get(key); - } - try { - const created = await store.attributename.create(googleAuthId, { - name: normalized, - exclusive: true, - }); - attributeMap.set(key, created.id); - return created.id; - } catch (err) { - log.warn(`Could not create attribute "${normalized}":`, err.message); - return null; - } - }; - - // Group children by parent SKU - const parentChildrenMap = new Map(); - for (const child of includedArticles) { - const parentJtlId = String(child.kVaterArtikel || '').trim(); - if (!parentJtlId || parentJtlId === '0') continue; - - const parentSku = jtlArtIdToSku.get(parentJtlId); - const childSku = cleanValue(child.cArtNr); - if (!parentSku || !childSku || parentSku === childSku) continue; - - if (!parentChildrenMap.has(parentSku)) { - const parentRow = includedArticles.find((a) => String(a.kArtikel) === parentJtlId) || null; - parentChildrenMap.set(parentSku, { parentRow, children: [] }); - } - parentChildrenMap.get(parentSku).children.push(child); - } - - let variantsLinked = 0; - for (const [parentSku, { parentRow, children }] of parentChildrenMap.entries()) { - const rawAttrName = parentRow ? cleanValue(parentRow.kEigenschaftKombi) : ''; - const targetAttrId = await resolveAttributeId(rawAttrName || 'Variante'); - - if (targetAttrId) { - try { - await store.article.toggleParent(googleAuthId, parentSku, { - isParent: true, - attributeNameId: targetAttrId, - }); - } catch (err) { - log.warn(`Could not toggle parent for ${parentSku}:`, err.message); - } - } - - const usedValues = new Set(); - for (const child of children) { - const childSku = cleanValue(child.cArtNr); - let val = cleanValue(child.kEigenschaftKombi) || 'Standard'; - - if (usedValues.has(val.toLowerCase())) { - val = `${val} (${childSku})`; - } - usedValues.add(val.toLowerCase()); - - try { - if (targetAttrId) { - await store.article.attachChild(googleAuthId, parentSku, { - sku: childSku, - value: val, - reparent: true, - }); - variantsLinked++; - } else { - await store.article.setParentBySku(googleAuthId, childSku, parentSku); - variantsLinked++; - } - } catch (err) { - log.warn(`Failed to attach child ${childSku} to parent ${parentSku}:`, err.message); - } - } - } - log.info(`Linked ${variantsLinked} variants across ${parentChildrenMap.size} parents`); - - // ─── Phase 5: Article Pictures Import ──────────────────────────────────── - let totalImagesImported = 0; - for (const task of articleImageTasks) { - try { - const imgBuf = fs.readFileSync(task.fullPath); - await store.article.addImage(googleAuthId, task.sku, imgBuf); - totalImagesImported++; - - const skele = await computeSkeleHash(imgBuf); - emitSkeleHash(skele?.base62 || null); - } catch (err) { - log.warn(`Article image add failed for ${task.sku} (${task.relPath}):`, err.message); - } - progressCounts.imagesdone++; - emitProgress(); - } - - flushSkeleHashes(); - emitProgress(true); - log.info(`Article pictures complete: ${totalImagesImported}/${articleImageTasks.length}`); - - // ─── Phase 6: Orders / Receipts (Manifests) Import ─────────────────────── - let ordersImported = 0; - let ordersSkipped = 0; - - if (manRecords.length > 0 && itemRecords.length > 0) { - const itemsByAuftrag = new Map(); - for (const it of itemRecords) { - const kAuftrag = String(it.kAuftrag || '').trim(); - if (!kAuftrag) continue; - if (!itemsByAuftrag.has(kAuftrag)) { - itemsByAuftrag.set(kAuftrag, []); - } - itemsByAuftrag.get(kAuftrag).push(it); - } - - const existingOrdersResult = await store.order.list(googleAuthId); - const existingOrderList = Array.isArray(existingOrdersResult) - ? existingOrdersResult - : (existingOrdersResult?.orders || []); - const existingOrderNumbers = new Set( - existingOrderList.map((o) => (o.number || '').trim()).filter(Boolean) - ); - - let defaultCustomerGroupId = null; - try { - const groups = await store.customergroup.list(googleAuthId); - defaultCustomerGroupId = groups[0]?.id || null; - } catch {} - - const entityMap = new Map(); - try { - const entities = await store.entity.list(googleAuthId); - for (const ent of entities) { - if (ent.name) { - entityMap.set(ent.name.trim().toLowerCase(), ent.id); - } - } - } catch {} - - const resolveEntityId = async (name, firstName) => { - const cleanName = (name || '').trim(); - if (!cleanName || !defaultCustomerGroupId) return null; - const key = cleanName.toLowerCase(); - if (entityMap.has(key)) return entityMap.get(key); - - try { - const ent = await store.entity.create(googleAuthId, { - customergroupId: defaultCustomerGroupId, - name: cleanName, - firstName: (firstName || '').trim() || undefined, - }); - entityMap.set(key, ent.id); - return ent.id; - } catch (err) { - log.warn(`Could not create entity "${cleanName}":`, err.message); - return null; - } - }; - - for (const m of manRecords) { - const orderNumber = cleanValue(m.cAuftragsNr); - if (!orderNumber || existingOrderNumbers.has(orderNumber)) { - ordersSkipped++; - progressCounts.receiptsdone++; - emitProgress(); - continue; - } - - const rawItems = itemsByAuftrag.get(String(m.kAuftrag || '').trim()) || []; - if (rawItems.length === 0) { - ordersSkipped++; - progressCounts.receiptsdone++; - emitProgress(); - continue; - } - - const lines = rawItems.map((it) => { - const sku = cleanValue(it.cArtNr) || null; - const rawName = cleanValue(it.cName || it.cNameStandard); - const name = rawName || (sku ? `Artikel ${sku}` : 'Artikel'); - const amount = parseGermanFloat(it.fAnzahl, 1); - const net = parseGermanFloat(it.fVKNetto, 0); - const taxrate = parseGermanFloat(it.fMwSt, 19); - const rebate = parseGermanFloat(it.fRabatt, 0); - const comment = cleanValue(it.cHinweis) || null; - - return { - sku, - name, - amount, - net, - taxrate, - rebate, - comment, - }; - }); - - const custName = cleanValue(m.cName) || 'Laufkunde'; - const custFirstName = cleanValue(m.cVorname); - const entityId = await resolveEntityId(custName, custFirstName); - - const dateStr = cleanValue(m.dErstellt); - const parsedDate = dateStr ? new Date(dateStr) : new Date(); - const validDate = !Number.isNaN(parsedDate.getTime()) ? parsedDate.toISOString() : new Date().toISOString(); - - try { - await store.order.create(googleAuthId, { - number: orderNumber, - externalNumber: cleanValue(m.cExterneAuftragsnummer) || null, - date: validDate, - createdAt: validDate, - entityId: entityId || undefined, - lines, - }); - existingOrderNumbers.add(orderNumber); - ordersImported++; - } catch (err) { - log.warn(`Failed to create order ${orderNumber}:`, err.message); - } - - progressCounts.receiptsdone++; - emitProgress(); - } - - emitProgress(true); - log.info(`Orders complete: ${ordersImported} created, ${ordersSkipped} skipped`); - } - - flushSkeleHashes(); - emitProgress(true); + // Emit baseline counts immediately + progressTracker.emitProgress(true); + // #endregion + + // MARK: - Phase 1: Categories + // #region Phase 1: Categories + const categoriesResult = await importCategories({ + store, + googleAuthId, + includedCategories, + categoryImageTasks, + progressTracker, + }); + // #endregion + + // MARK: - Phase 2: Brands + // #region Phase 2: Brands + const brandsResult = await importBrands({ + store, + googleAuthId, + brandMap, + brandImageTasks, + progressTracker, + }); + // #endregion + + // MARK: - Phase 3: Articles + // #region Phase 3: Articles + const articlesResult = await importArticles({ + store, + googleAuthId, + includedArticles, + jtlCatIdToDashCatId: categoriesResult.jtlCatIdToDashCatId, + hauptDashId: categoriesResult.hauptDashId, + brandNameToId: brandsResult.brandNameToId, + progressTracker, + }); + // #endregion + + // MARK: - Phase 4: Variants + // #region Phase 4: Variants + const variantsResult = await importVariants({ + store, + googleAuthId, + includedArticles, + jtlArtIdToSku: articlesResult.jtlArtIdToSku, + }); + // #endregion + + // MARK: - Phase 5: Article Pictures + // #region Phase 5: Article Pictures + const imagesResult = await importArticleImages({ + store, + googleAuthId, + articleImageTasks, + progressTracker, + }); + // #endregion + + // MARK: - Phase 6: Orders / Receipts + // #region Phase 6: Orders / Receipts + const ordersResult = await importOrders({ + store, + googleAuthId, + manRecords, + itemRecords, + progressTracker, + }); + // #endregion + + // Flush any pending images and progress updates + progressTracker.flushSkeleHashes(); + progressTracker.emitProgress(true); return { ok: true, message: 'Import erfolgreich abgeschlossen.', stats: { - categories: jtlCatIdToDashCatId.size, - categoriesCreated: createdCatCount, - categoryPictures: catPicCount, - brands: brandNameToId.size, - brandsCreated: createdBrandCount, - brandPictures: brandPicCount, + categories: categoriesResult.jtlCatIdToDashCatId.size, + categoriesCreated: categoriesResult.createdCatCount, + categoryPictures: categoriesResult.catPicCount, + brands: brandsResult.brandNameToId.size, + brandsCreated: brandsResult.createdBrandCount, + brandPictures: brandsResult.brandPicCount, articlesTotal: includedArticles.length, - articlesImported, - articlesUpdated, - variantsLinked, - articlePictures: totalImagesImported, + articlesImported: articlesResult.articlesImported, + articlesUpdated: articlesResult.articlesUpdated, + variantsLinked: variantsResult.variantsLinked, + articlePictures: imagesResult.totalImagesImported, ordersTotal: totalReceiptsCount, - ordersImported, - ordersSkipped, + ordersImported: ordersResult.ordersImported, + ordersSkipped: ordersResult.ordersSkipped, }, }; } + +// #endregion diff --git a/src/services/companyImport/articleImageImporter.js b/src/services/companyImport/articleImageImporter.js new file mode 100644 index 0000000..f01d082 --- /dev/null +++ b/src/services/companyImport/articleImageImporter.js @@ -0,0 +1,55 @@ +// MARK: - Imports +// #region Imports +import fs from 'fs'; +import { computeSkeleHash } from './imageHasher.js'; +import { createLogger } from '../../util/logger.js'; +// #endregion + +const log = createLogger('import'); + +// MARK: - Article Image Importer +// #region Article Image Importer + +/** + * Imports article images and emits their skelehashes. + * + * @param {object} params + * @param {object} params.store - Store singleton + * @param {number} params.googleAuthId + * @param {Array} params.articleImageTasks + * @param {object} params.progressTracker + * @returns {Promise<{ totalImagesImported: number }>} + */ +export async function importArticleImages({ + store, + googleAuthId, + articleImageTasks, + progressTracker, +}) { + let totalImagesImported = 0; + + for (const task of articleImageTasks) { + try { + const imgBuf = fs.readFileSync(task.fullPath); + await store.article.addImage(googleAuthId, task.sku, imgBuf); + totalImagesImported++; + + const skele = await computeSkeleHash(imgBuf); + progressTracker.emitSkeleHash(skele?.base62 || null); + } catch (err) { + log.warn(`Article image add failed for ${task.sku} (${task.relPath}):`, err.message); + } + progressTracker.counts.imagesdone++; + progressTracker.emitProgress(); + } + + progressTracker.flushSkeleHashes(); + progressTracker.emitProgress(true); + log.info(`Article pictures complete: ${totalImagesImported}/${articleImageTasks.length}`); + + return { + totalImagesImported, + }; +} + +// #endregion diff --git a/src/services/companyImport/articleImporter.js b/src/services/companyImport/articleImporter.js new file mode 100644 index 0000000..67bd627 --- /dev/null +++ b/src/services/companyImport/articleImporter.js @@ -0,0 +1,190 @@ +// MARK: - Imports +// #region Imports +import { cleanValue, parseGermanFloat } from './csvParser.js'; +import { createLogger } from '../../util/logger.js'; +// #endregion + +const log = createLogger('import'); + +// MARK: - Article Importer +// #region Article Importer + +/** + * Imports articles (insert or update), setting prices, categories, barcodes, and brands. + * + * @param {object} params + * @param {object} params.store - Store singleton + * @param {number} params.googleAuthId + * @param {Array>} params.includedArticles + * @param {Map} params.jtlCatIdToDashCatId + * @param {number} params.hauptDashId + * @param {Map} params.brandNameToId + * @param {object} params.progressTracker + * @returns {Promise<{ jtlArtIdToSku: Map, articlesImported: number, articlesUpdated: number }>} + */ +export async function importArticles({ + store, + googleAuthId, + includedArticles, + jtlCatIdToDashCatId, + hauptDashId, + brandNameToId, + progressTracker, +}) { + const jtlArtIdToSku = new Map(); + let articlesImported = 0; + let articlesUpdated = 0; + + for (const a of includedArticles) { + const jtlArtId = String(a.kArtikel); + const sku = cleanValue(a.cArtNr); + if (!sku) { + progressTracker.counts.articlesdone++; + continue; + } + + jtlArtIdToSku.set(jtlArtId, sku); + + let name = cleanValue(a.cName); + let desc = cleanValue(a.descriptions); + + if (!name && desc) { + try { + const parsed = JSON.parse(desc); + const item = Array.isArray(parsed) ? parsed[0] : parsed; + if (item?.cName) name = String(item.cName).trim(); + if (item?.cBeschreibung) desc = String(item.cBeschreibung).trim(); + } catch {} + } + if (!name) name = `Artikel ${sku}`; + + // Category mapping + const assignedDashCatIds = []; + const rawCatStr = String(a.categories || '').trim(); + if (rawCatStr) { + let parsedIds = []; + if (rawCatStr.startsWith('[') || rawCatStr.startsWith('{')) { + try { + const raw = JSON.parse(rawCatStr); + parsedIds = (Array.isArray(raw) ? raw : [raw]).map((it) => String(it.kKategorie || it)); + } catch { + parsedIds = rawCatStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean); + } + } else { + parsedIds = rawCatStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean); + } + for (const jtlCId of parsedIds) { + if (jtlCatIdToDashCatId.has(jtlCId)) { + assignedDashCatIds.push(jtlCatIdToDashCatId.get(jtlCId)); + } + } + } + const primaryCatId = assignedDashCatIds[0] || hauptDashId; + + const trackStock = a.cLagerAktiv === 'Y' ? 1 : 0; + const netPrice = parseGermanFloat(a.fVKNetto, 0); + const taxRate = a.kSteuerklasse === '3' ? 7 : 19; + const barcode = cleanValue(a.cBarcode); + + const hersteller = cleanValue(a.Hersteller); + const brandId = hersteller ? (brandNameToId.get(hersteller.toLowerCase()) || null) : null; + + let existingArticle = null; + try { + const res = await store.article.getBySku(googleAuthId, sku); + existingArticle = res.article; + } catch { + existingArticle = null; + } + + try { + if (existingArticle) { + await store.article.updateBySku(googleAuthId, sku, { + name, + track_stock: trackStock, + desc: desc || undefined, + }); + + await store.article.setPricesBySku(googleAuthId, sku, [ + { net: netPrice, tax: taxRate }, + ]); + + if (assignedDashCatIds.length > 0) { + await store.article.setCategoryIdsBySku(googleAuthId, sku, assignedDashCatIds); + } + + if (barcode) { + try { + const currentEans = existingArticle.eans || []; + if (!currentEans.includes(barcode)) { + await store.article.setEansBySku(googleAuthId, sku, [...currentEans, barcode]); + } + } catch (eanErr) { + log.warn(`Could not set EAN for ${sku}:`, eanErr.message); + } + } + + if (brandId && existingArticle.brand?.id !== brandId) { + try { + await store.article.setBrandIdBySku(googleAuthId, sku, brandId); + } catch (bErr) { + log.warn(`Could not set brand for ${sku}:`, bErr.message); + } + } + + articlesUpdated++; + } else { + await store.article.createUnderParent({ + googleAuthId, + parentId: primaryCatId, + name, + sku, + track_stock: trackStock, + desc: desc || undefined, + }); + + await store.article.setPricesBySku(googleAuthId, sku, [ + { net: netPrice, tax: taxRate }, + ]); + + if (assignedDashCatIds.length > 1) { + await store.article.setCategoryIdsBySku(googleAuthId, sku, assignedDashCatIds); + } + + if (barcode) { + try { + await store.article.setEansBySku(googleAuthId, sku, [barcode]); + } catch (eanErr) { + log.warn(`Could not set EAN for ${sku}:`, eanErr.message); + } + } + + if (brandId) { + try { + await store.article.setBrandIdBySku(googleAuthId, sku, brandId); + } catch (bErr) { + log.warn(`Could not set brand for ${sku}:`, bErr.message); + } + } + + articlesImported++; + } + } catch (err) { + log.warn(`Failed to save article ${sku} ("${name}"):`, err.message); + } + + progressTracker.counts.articlesdone++; + progressTracker.emitProgress(); + } + + progressTracker.emitProgress(true); + log.info(`Articles complete: New: ${articlesImported}, Updated: ${articlesUpdated}`); + + return { + jtlArtIdToSku, + articlesImported, + articlesUpdated, + }; +} + +// #endregion diff --git a/src/services/companyImport/brandImporter.js b/src/services/companyImport/brandImporter.js new file mode 100644 index 0000000..67a9eb3 --- /dev/null +++ b/src/services/companyImport/brandImporter.js @@ -0,0 +1,91 @@ +// MARK: - Imports +// #region Imports +import fs from 'fs'; +import { computeSkeleHash } from './imageHasher.js'; +import { createLogger } from '../../util/logger.js'; +// #endregion + +const log = createLogger('import'); + +// MARK: - Brand Importer +// #region Brand Importer + +/** + * Imports brand manufacturers and attaches brand logos. + * + * @param {object} params + * @param {object} params.store - Store singleton + * @param {number} params.googleAuthId + * @param {Map>} params.brandMap + * @param {Array} params.brandImageTasks + * @param {object} params.progressTracker + * @returns {Promise<{ brandNameToId: Map, createdBrandCount: number, brandPicCount: number }>} + */ +export async function importBrands({ + store, + googleAuthId, + brandMap, + brandImageTasks, + progressTracker, +}) { + const brandNameToId = new Map(); + let createdBrandCount = 0; + let brandPicCount = 0; + + const { brands: existingBrandRows } = await store.brand.list(googleAuthId, { offset: 0, limit: 100 }); + const existingBrandNames = new Set( + existingBrandRows.map((b) => (b.name || '').trim().toLowerCase()).filter(Boolean) + ); + + const brandTaskMap = new Map(); + for (const task of brandImageTasks) { + brandTaskMap.set(task.brandName.toLowerCase(), task); + } + + for (const [brandName] of brandMap.entries()) { + const lowerName = brandName.toLowerCase(); + const isNew = !existingBrandNames.has(lowerName); + + try { + const brand = await store.brand.upsert(googleAuthId, { name: brandName }); + brandNameToId.set(lowerName, brand.id); + if (isNew) { + createdBrandCount++; + existingBrandNames.add(lowerName); + } + + const imgTask = brandTaskMap.get(lowerName); + if (imgTask) { + if (!brand.pics || brand.pics.length === 0) { + try { + const imgBuf = fs.readFileSync(imgTask.fullPath); + await store.brand.addImage(googleAuthId, brand.id, imgBuf); + brandPicCount++; + + const skele = await computeSkeleHash(imgBuf); + progressTracker.emitSkeleHash(skele?.base62 || null); + } catch (err) { + log.warn(`Brand image error for "${brandName}":`, err.message); + } + } + progressTracker.counts.imagesdone++; + } + } catch (err) { + log.warn(`Failed to upsert brand "${brandName}":`, err.message); + } + + progressTracker.emitProgress(); + } + + progressTracker.flushSkeleHashes(); + progressTracker.emitProgress(true); + log.info(`Brands resolved: ${brandNameToId.size}, created: ${createdBrandCount}, images: ${brandPicCount}`); + + return { + brandNameToId, + createdBrandCount, + brandPicCount, + }; +} + +// #endregion diff --git a/src/services/companyImport/catalogPreprocessor.js b/src/services/companyImport/catalogPreprocessor.js new file mode 100644 index 0000000..b32762e --- /dev/null +++ b/src/services/companyImport/catalogPreprocessor.js @@ -0,0 +1,170 @@ +// MARK: - Imports +// #region Imports +import fs from 'fs'; +import path from 'path'; +import { cleanValue, parseCsvRecords } from './csvParser.js'; +// #endregion + +// MARK: - Category & Article Filtering +// #region Category & Article Filtering + +/** + * Filters out trash/papierkorb categories and their descendants, + * as well as articles that exclusively belong to excluded categories. + * + * @param {Array>} catRecords + * @param {Array>} artRecords + * @returns {{ includedCategories: Array>, includedArticles: Array>, excludedJtlIds: Set }} + */ +export function filterCatalogRecords(catRecords, artRecords) { + const excludedJtlIds = new Set(['72']); + + for (const c of catRecords) { + let name = (c.cName || '').trim(); + if (!name) { + try { + const tr = JSON.parse(c.translations || '[]'); + name = tr[0]?.cName || ''; + } catch {} + } + if (name.toLowerCase().includes('papierkorb') || name.toLowerCase().includes('trash')) { + excludedJtlIds.add(String(c.kKategorie)); + } + } + + let addedExcluded = true; + while (addedExcluded) { + addedExcluded = false; + for (const c of catRecords) { + const id = String(c.kKategorie); + const parent = String(c.kOberKategorie); + if (!excludedJtlIds.has(id) && excludedJtlIds.has(parent)) { + excludedJtlIds.add(id); + addedExcluded = true; + } + } + } + + const includedCategories = catRecords.filter((c) => !excludedJtlIds.has(String(c.kKategorie))); + + const includedArticles = artRecords.filter((a) => { + const catStr = String(a.categories || '').trim(); + if (!catStr) return true; + let catIds = []; + if (catStr.startsWith('[') || catStr.startsWith('{')) { + try { + const raw = JSON.parse(catStr); + catIds = (Array.isArray(raw) ? raw : [raw]).map((it) => String(it.kKategorie || it)); + } catch { + catIds = catStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean); + } + } else { + catIds = catStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean); + } + if (catIds.length > 0 && catIds.every((cid) => excludedJtlIds.has(cid))) { + return false; + } + return true; + }); + + return { includedCategories, includedArticles, excludedJtlIds }; +} + +// #endregion + +// MARK: - Task Collection & Manifests +// #region Task Collection & Manifests + +/** + * Scans directories for existing category, brand, and article images, + * and loads receipts / manifest records if available. + * + * @param {object} params + * @param {string} params.importDir + * @param {Array>} params.includedCategories + * @param {Array>} params.includedArticles + * @returns {object} Discovered image tasks, brand map, and manifest records + */ +export function prepareImportTasks({ importDir, includedCategories, includedArticles }) { + // Category images + const categoryImageTasks = []; + for (const c of includedCategories) { + const picRel = (c.pictureFiles || '').trim(); + if (picRel) { + for (const rel of picRel.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean)) { + const full = path.resolve(importDir, rel); + if (fs.existsSync(full)) { + categoryImageTasks.push({ catJtlId: String(c.kKategorie), relPath: rel, fullPath: full }); + } + } + } + } + + // Brand images (unique manufacturers, 1 primary logo) + const brandMap = new Map(); // brandName -> array of { relPath, fullPath } + for (const a of includedArticles) { + const brand = cleanValue(a.Hersteller); + if (!brand) continue; + if (!brandMap.has(brand)) { + brandMap.set(brand, []); + const rawPics = String(a.manufacturerPictureFiles || '').trim(); + if (rawPics) { + for (const rel of rawPics.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean)) { + const full = path.resolve(importDir, rel); + if (fs.existsSync(full)) { + brandMap.get(brand).push({ relPath: rel, fullPath: full }); + } + } + } + } + } + + const brandImageTasks = []; + for (const [brandName, pics] of brandMap.entries()) { + if (pics.length > 0) { + brandImageTasks.push({ brandName, ...pics[0] }); + } + } + + // Article images + const articleImageTasks = []; + for (const a of includedArticles) { + const sku = cleanValue(a.cArtNr); + if (!sku) continue; + const rawPics = String(a.pictureFiles || '').trim(); + if (rawPics) { + for (const rel of rawPics.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean)) { + const full = path.resolve(importDir, rel); + if (fs.existsSync(full)) { + articleImageTasks.push({ sku, relPath: rel, fullPath: full }); + } + } + } + } + + const totalImagesCount = categoryImageTasks.length + brandImageTasks.length + articleImageTasks.length; + + // Receipts / Manifests + const manCsvPath = path.join(importDir, 'manifests.csv'); + const itemsCsvPath = path.join(importDir, 'manifestitems.csv'); + let manRecords = []; + let itemRecords = []; + if (fs.existsSync(manCsvPath) && fs.existsSync(itemsCsvPath)) { + manRecords = parseCsvRecords(fs.readFileSync(manCsvPath, 'utf8')); + itemRecords = parseCsvRecords(fs.readFileSync(itemsCsvPath, 'utf8')); + } + const totalReceiptsCount = manRecords.length; + + return { + categoryImageTasks, + brandMap, + brandImageTasks, + articleImageTasks, + totalImagesCount, + manRecords, + itemRecords, + totalReceiptsCount, + }; +} + +// #endregion diff --git a/src/services/companyImport/categoryImporter.js b/src/services/companyImport/categoryImporter.js new file mode 100644 index 0000000..8323f0e --- /dev/null +++ b/src/services/companyImport/categoryImporter.js @@ -0,0 +1,148 @@ +// MARK: - Imports +// #region Imports +import fs from 'fs'; +import { cleanValue } from './csvParser.js'; +import { computeSkeleHash } from './imageHasher.js'; +import { createLogger } from '../../util/logger.js'; +// #endregion + +const log = createLogger('import'); + +// MARK: - Category Importer +// #region Category Importer + +/** + * Imports category tree and attaches category images. + * + * @param {object} params + * @param {object} params.store - Store singleton + * @param {number} params.googleAuthId + * @param {Array>} params.includedCategories + * @param {Array} params.categoryImageTasks + * @param {object} params.progressTracker + * @returns {Promise<{ jtlCatIdToDashCatId: Map, hauptDashId: number, createdCatCount: number, catPicCount: number }>} + */ +export async function importCategories({ + store, + googleAuthId, + includedCategories, + categoryImageTasks, + progressTracker, +}) { + const treeResult = await store.category.getTreeForCompany(googleAuthId); + const existingTree = treeResult.tree || []; + const rootCategory = existingTree.find((c) => c.parentId == null) || existingTree[0]; + if (!rootCategory) { + throw new Error('Company root category not found'); + } + const rootCategoryId = rootCategory.id; + + const existingMap = new Map(); + for (const c of existingTree) { + const key = `${c.parentId ?? 'root'}:${String(c.name || '').trim().toLowerCase()}`; + existingMap.set(key, c.id); + } + + const jtlCatIdToDashCatId = new Map(); + jtlCatIdToDashCatId.set('0', rootCategoryId); + + // Group category images by JTL category ID + const catImagesByJtlId = new Map(); + for (const task of categoryImageTasks) { + if (!catImagesByJtlId.has(task.catJtlId)) { + catImagesByJtlId.set(task.catJtlId, []); + } + catImagesByJtlId.get(task.catJtlId).push(task); + } + + const pendingCats = [...includedCategories]; + let createdCatCount = 0; + let catPicCount = 0; + let safety = 0; + + while (pendingCats.length > 0 && safety < 10) { + safety++; + const remaining = []; + for (const c of pendingCats) { + const jtlId = String(c.kKategorie); + const jtlParent = String(c.kOberKategorie || '0'); + + let name = cleanValue(c.cName); + if (!name) { + try { + const tr = JSON.parse(c.translations || '[]'); + name = tr[0]?.cName || ''; + } catch {} + } + if (!name) name = `Kategorie ${jtlId}`; + + let dashParentId; + if (jtlParent === '0' || !jtlParent) { + dashParentId = rootCategoryId; + } else if (jtlCatIdToDashCatId.has(jtlParent)) { + dashParentId = jtlCatIdToDashCatId.get(jtlParent); + } else { + remaining.push(c); + continue; + } + + const key = `${dashParentId}:${name.toLowerCase()}`; + let dashCatId = existingMap.get(key); + + if (!dashCatId) { + try { + const created = await store.category.createUnderParent({ + googleAuthId, + parentId: dashParentId, + name, + }); + dashCatId = created.id; + existingMap.set(key, dashCatId); + createdCatCount++; + } catch (err) { + log.warn(`Failed to create category "${name}":`, err.message); + } + } + + if (dashCatId) { + jtlCatIdToDashCatId.set(jtlId, dashCatId); + + // Add category images + const imgs = catImagesByJtlId.get(jtlId) || []; + for (const imgTask of imgs) { + try { + const imgBuf = fs.readFileSync(imgTask.fullPath); + await store.category.addImage(googleAuthId, dashCatId, imgBuf); + catPicCount++; + + const skele = await computeSkeleHash(imgBuf); + progressTracker.emitSkeleHash(skele?.base62 || null); + } catch (err) { + log.warn(`Category image error (${imgTask.relPath}):`, err.message); + } + progressTracker.counts.imagesdone++; + } + } + + progressTracker.counts.categoriesdone++; + progressTracker.emitProgress(); + } + pendingCats.length = 0; + pendingCats.push(...remaining); + } + + progressTracker.flushSkeleHashes(); + progressTracker.emitProgress(true); + log.info(`Categories resolved: ${jtlCatIdToDashCatId.size}/${includedCategories.length}, created: ${createdCatCount}, images: ${catPicCount}`); + + const hauptDashId = jtlCatIdToDashCatId.get('1') || rootCategoryId; + + return { + jtlCatIdToDashCatId, + hauptDashId, + createdCatCount, + catPicCount, + }; +} + +// #endregion diff --git a/src/services/companyImport/csvParser.js b/src/services/companyImport/csvParser.js new file mode 100644 index 0000000..e132f2b --- /dev/null +++ b/src/services/companyImport/csvParser.js @@ -0,0 +1,128 @@ +// MARK: - CSV Parsing & Value Sanitization +// #region CSV Parsing & Value Sanitization + +/** + * Cleans formula prefixes (="..."), quotes, and whitespace from CSV values. + * + * @param {string|any} str + * @returns {string} + */ +export function cleanValue(str) { + if (str == null) return ''; + let val = String(str).trim(); + if (val.startsWith('=')) { + val = val.slice(1).trim(); + } + if (val.startsWith('"') && val.endsWith('"')) { + val = val.slice(1, -1).trim(); + } + return val; +} + +/** + * Parses German decimal format numbers (e.g. "8,36" -> 8.36). + * + * @param {string|number} str + * @param {number} [fallback=0] + * @returns {number} + */ +export function parseGermanFloat(str, fallback = 0) { + const clean = cleanValue(str).replace(',', '.'); + const num = parseFloat(clean); + return Number.isNaN(num) ? fallback : num; +} + +/** + * Parses CSV text into an array of objects keyed by header. + * Supports auto-detecting semicolon (;) vs comma (,) delimiters, + * quoted fields with embedded newlines/quotes, and formula prefixes. + * + * @param {string} text - Raw CSV text + * @returns {Array>} + */ +export function parseCsvRecords(text) { + const clean = text.replace(/^\uFEFF/, ''); + const firstLine = clean.split(/[\r\n]/)[0] || ''; + const semiCount = (firstLine.match(/;/g) || []).length; + const commaCount = (firstLine.match(/,/g) || []).length; + const delimiter = semiCount >= commaCount ? ';' : ','; + + const rows = []; + let field = ''; + let row = []; + let inQuotes = false; + let i = 0; + const len = clean.length; + + while (i < len) { + const c = clean[i]; + if (inQuotes) { + if (c === '"') { + if (i + 1 < len && clean[i + 1] === '"') { + field += '"'; + i += 2; + continue; + } else { + inQuotes = false; + i++; + continue; + } + } else { + field += c; + i++; + continue; + } + } else { + if (c === '"') { + inQuotes = true; + i++; + continue; + } else if (c === delimiter) { + row.push(cleanValue(field)); + field = ''; + i++; + continue; + } else if (c === '\r') { + if (i + 1 < len && clean[i + 1] === '\n') { + i++; + } + row.push(cleanValue(field)); + rows.push(row); + field = ''; + row = []; + i++; + continue; + } else if (c === '\n') { + row.push(cleanValue(field)); + rows.push(row); + field = ''; + row = []; + i++; + continue; + } else { + field += c; + i++; + continue; + } + } + } + if (field !== '' || row.length > 0) { + row.push(cleanValue(field)); + rows.push(row); + } + if (rows.length === 0) return []; + const header = rows[0].map((h) => cleanValue(h)); + const records = []; + for (let r = 1; r < rows.length; r++) { + const curr = rows[r]; + if (curr.length === 1 && curr[0] === '') continue; + const obj = {}; + for (let col = 0; col < header.length; col++) { + obj[header[col]] = curr[col] ?? ''; + } + records.push(obj); + } + return records; +} + +// #endregion diff --git a/src/services/companyImport/imageHasher.js b/src/services/companyImport/imageHasher.js new file mode 100644 index 0000000..b304bb3 --- /dev/null +++ b/src/services/companyImport/imageHasher.js @@ -0,0 +1,41 @@ +// MARK: - Imports +// #region Imports +import sharp from 'sharp'; +import { encodeRGBA } from 'skelehash-wasm'; +import { createLogger } from '../../util/logger.js'; +// #endregion + +const log = createLogger('import'); + +// MARK: - SkeleHash Computation +// #region SkeleHash Computation + +/** + * Computes SkeleHash base62 hash for an image buffer. + * + * @param {Buffer} buffer + * @returns {Promise<{base62: string, width: number, height: number, hashLength: number}|null>} + */ +export async function computeSkeleHash(buffer) { + try { + const { data, info } = await sharp(buffer) + .rotate() + .resize({ width: 200, height: 200, fit: 'inside' }) + .ensureAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); + + const result = await encodeRGBA(info.width, info.height, data); + return { + base62: result.base62, + width: info.width, + height: info.height, + hashLength: result.hash?.length ?? 0, + }; + } catch (err) { + log.warn('Failed to compute skelehash for image:', err.message); + return null; + } +} + +// #endregion diff --git a/src/services/companyImport/orderImporter.js b/src/services/companyImport/orderImporter.js new file mode 100644 index 0000000..c01ce29 --- /dev/null +++ b/src/services/companyImport/orderImporter.js @@ -0,0 +1,165 @@ +// MARK: - Imports +// #region Imports +import { cleanValue, parseGermanFloat } from './csvParser.js'; +import { createLogger } from '../../util/logger.js'; +// #endregion + +const log = createLogger('import'); + +// MARK: - Order / Manifest Importer +// #region Order / Manifest Importer + +/** + * Imports order manifests and line items, resolving customer entities. + * + * @param {object} params + * @param {object} params.store - Store singleton + * @param {number} params.googleAuthId + * @param {Array>} params.manRecords + * @param {Array>} params.itemRecords + * @param {object} params.progressTracker + * @returns {Promise<{ ordersImported: number, ordersSkipped: number }>} + */ +export async function importOrders({ + store, + googleAuthId, + manRecords, + itemRecords, + progressTracker, +}) { + let ordersImported = 0; + let ordersSkipped = 0; + + if (manRecords.length === 0 || itemRecords.length === 0) { + return { ordersImported, ordersSkipped }; + } + + const itemsByAuftrag = new Map(); + for (const it of itemRecords) { + const kAuftrag = String(it.kAuftrag || '').trim(); + if (!kAuftrag) continue; + if (!itemsByAuftrag.has(kAuftrag)) { + itemsByAuftrag.set(kAuftrag, []); + } + itemsByAuftrag.get(kAuftrag).push(it); + } + + const existingOrdersResult = await store.order.list(googleAuthId); + const existingOrderList = Array.isArray(existingOrdersResult) + ? existingOrdersResult + : (existingOrdersResult?.orders || []); + const existingOrderNumbers = new Set( + existingOrderList.map((o) => (o.number || '').trim()).filter(Boolean) + ); + + let defaultCustomerGroupId = null; + try { + const groups = await store.customergroup.list(googleAuthId); + defaultCustomerGroupId = groups[0]?.id || null; + } catch {} + + const entityMap = new Map(); + try { + const entities = await store.entity.list(googleAuthId); + for (const ent of entities) { + if (ent.name) { + entityMap.set(ent.name.trim().toLowerCase(), ent.id); + } + } + } catch {} + + const resolveEntityId = async (name, firstName) => { + const cleanName = (name || '').trim(); + if (!cleanName || !defaultCustomerGroupId) return null; + const key = cleanName.toLowerCase(); + if (entityMap.has(key)) return entityMap.get(key); + + try { + const ent = await store.entity.create(googleAuthId, { + customergroupId: defaultCustomerGroupId, + name: cleanName, + firstName: (firstName || '').trim() || undefined, + }); + entityMap.set(key, ent.id); + return ent.id; + } catch (err) { + log.warn(`Could not create entity "${cleanName}":`, err.message); + return null; + } + }; + + for (const m of manRecords) { + const orderNumber = cleanValue(m.cAuftragsNr); + if (!orderNumber || existingOrderNumbers.has(orderNumber)) { + ordersSkipped++; + progressTracker.counts.receiptsdone++; + progressTracker.emitProgress(); + continue; + } + + const rawItems = itemsByAuftrag.get(String(m.kAuftrag || '').trim()) || []; + if (rawItems.length === 0) { + ordersSkipped++; + progressTracker.counts.receiptsdone++; + progressTracker.emitProgress(); + continue; + } + + const lines = rawItems.map((it) => { + const sku = cleanValue(it.cArtNr) || null; + const rawName = cleanValue(it.cName || it.cNameStandard); + const name = rawName || (sku ? `Artikel ${sku}` : 'Artikel'); + const amount = parseGermanFloat(it.fAnzahl, 1); + const net = parseGermanFloat(it.fVKNetto, 0); + const taxrate = parseGermanFloat(it.fMwSt, 19); + const rebate = parseGermanFloat(it.fRabatt, 0); + const comment = cleanValue(it.cHinweis) || null; + + return { + sku, + name, + amount, + net, + taxrate, + rebate, + comment, + }; + }); + + const custName = cleanValue(m.cName) || 'Laufkunde'; + const custFirstName = cleanValue(m.cVorname); + const entityId = await resolveEntityId(custName, custFirstName); + + const dateStr = cleanValue(m.dErstellt); + const parsedDate = dateStr ? new Date(dateStr) : new Date(); + const validDate = !Number.isNaN(parsedDate.getTime()) ? parsedDate.toISOString() : new Date().toISOString(); + + try { + await store.order.create(googleAuthId, { + number: orderNumber, + externalNumber: cleanValue(m.cExterneAuftragsnummer) || null, + date: validDate, + createdAt: validDate, + entityId: entityId || undefined, + lines, + }); + existingOrderNumbers.add(orderNumber); + ordersImported++; + } catch (err) { + log.warn(`Failed to create order ${orderNumber}:`, err.message); + } + + progressTracker.counts.receiptsdone++; + progressTracker.emitProgress(); + } + + progressTracker.emitProgress(true); + log.info(`Orders complete: ${ordersImported} created, ${ordersSkipped} skipped`); + + return { + ordersImported, + ordersSkipped, + }; +} + +// #endregion diff --git a/src/services/companyImport/progressTracker.js b/src/services/companyImport/progressTracker.js new file mode 100644 index 0000000..1d665f5 --- /dev/null +++ b/src/services/companyImport/progressTracker.js @@ -0,0 +1,74 @@ +// MARK: - Progress & Image Stream Tracking +// #region Progress & Image Stream Tracking + +/** + * Creates an event tracking controller for throttled progress updates and batched skelehashes. + * + * @param {object} params + * @param {object} params.initialCounts - Initial totals and completed counts + * @param {function} [params.onProgress] - Throttled callback for progress counts + * @param {function} [params.onImages] - Batched callback for emitted skelehash base62 strings + * @param {number} [params.progressThrottleMs=200] + * @param {number} [params.imageBatchMs=50] + * @returns {object} Tracker with mutateable counts and emission methods + */ +export function createProgressTracker({ + initialCounts, + onProgress = () => {}, + onImages = () => {}, + progressThrottleMs = 200, + imageBatchMs = 50, +}) { + const counts = { + categoriestotal: 0, + categoriesdone: 0, + articlestotal: 0, + articlesdone: 0, + imagestotal: 0, + imagesdone: 0, + receiptstotal: 0, + receiptsdone: 0, + ...initialCounts, + }; + + let lastProgressEmit = 0; + const emitProgress = (force = false) => { + const now = Date.now(); + if (force || now - lastProgressEmit >= progressThrottleMs) { + lastProgressEmit = now; + onProgress({ ...counts }); + } + }; + + let skeleBuffer = []; + let lastSkeleEmit = Date.now(); + + const emitSkeleHash = (base62, force = false) => { + if (base62) skeleBuffer.push(base62); + const now = Date.now(); + if (skeleBuffer.length > 0 && (force || now - lastSkeleEmit >= imageBatchMs)) { + const batch = skeleBuffer; + skeleBuffer = []; + lastSkeleEmit = now; + onImages(batch); + } + }; + + const flushSkeleHashes = () => { + if (skeleBuffer.length > 0) { + const batch = skeleBuffer; + skeleBuffer = []; + lastSkeleEmit = Date.now(); + onImages(batch); + } + }; + + return { + counts, + emitProgress, + emitSkeleHash, + flushSkeleHashes, + }; +} + +// #endregion diff --git a/src/services/companyImport/variantHelpers.js b/src/services/companyImport/variantHelpers.js new file mode 100644 index 0000000..2ae1641 --- /dev/null +++ b/src/services/companyImport/variantHelpers.js @@ -0,0 +1,53 @@ +// MARK: - Variant Attribute & Value Helpers +// #region Variant Attribute & Value Helpers + +/** + * Normalizes common variant attribute names and typos. + * + * @param {string} name + * @returns {string} + */ +export function normalizeAttributeName(name) { + const clean = (name || '').trim(); + if (!clean) return 'Variante'; + const lower = clean.toLowerCase(); + if (lower === 'menge' || lower === 'mange' || lower === 'meneg') return 'Menge'; + if (lower === 'farbe') return 'Farbe'; + if (lower === 'größe' || lower === 'grosse') return 'Größe'; + if (lower === 'sorte') return 'Sorte'; + if (lower === 'geschmack') return 'Geschmack'; + if (lower === 'typ') return 'Typ'; + if (lower === 'stück' || lower === 'stueck') return 'Stück'; + if (lower === 'ausführung' || lower === 'ausfuehrung') return 'Ausführung'; + if (lower === 'motiv') return 'Motiv'; + return clean.charAt(0).toUpperCase() + clean.slice(1); +} + +/** + * Fallback variant value extraction if child has no explicit attribute value. + * + * @param {string} parentName + * @param {string} childName + * @param {string} childSku + * @param {string} parentSku + * @returns {string} + */ +export function extractVariantValue(parentName, childName, childSku, parentSku) { + let pName = (parentName || '').replace(/[©®™]/g, '').trim(); + let cName = (childName || '').replace(/[©®™]/g, '').trim(); + + if (pName && cName) { + if (cName.toLowerCase().startsWith(pName.toLowerCase())) { + let val = cName.slice(pName.length).trim().replace(/^[-–—:/,\s]+/, '').trim(); + if (val) return val; + } + } + if (parentSku && childSku && childSku.startsWith(parentSku)) { + let suffix = childSku.slice(parentSku.length).trim().replace(/^[-_]+/, '').trim(); + if (suffix) return suffix; + } + if (cName && cName !== pName) return cName; + return childSku || 'Standard'; +} + +// #endregion diff --git a/src/services/companyImport/variantImporter.js b/src/services/companyImport/variantImporter.js new file mode 100644 index 0000000..041238e --- /dev/null +++ b/src/services/companyImport/variantImporter.js @@ -0,0 +1,129 @@ +// MARK: - Imports +// #region Imports +import { cleanValue } from './csvParser.js'; +import { normalizeAttributeName } from './variantHelpers.js'; +import { createLogger } from '../../util/logger.js'; +// #endregion + +const log = createLogger('import'); + +// MARK: - Variant Importer +// #region Variant Importer + +/** + * Resolves attribute definitions and links variant child articles to parents. + * + * @param {object} params + * @param {object} params.store - Store singleton + * @param {number} params.googleAuthId + * @param {Array>} params.includedArticles + * @param {Map} params.jtlArtIdToSku + * @returns {Promise<{ variantsLinked: number, parentsCount: number }>} + */ +export async function importVariants({ + store, + googleAuthId, + includedArticles, + jtlArtIdToSku, +}) { + const attributeMap = new Map(); + try { + const attrList = await store.attributename.list(googleAuthId); + for (const attr of attrList) { + if (attr.name) { + attributeMap.set(attr.name.trim().toLowerCase(), attr.id); + } + } + } catch (err) { + log.warn('Could not list attributes:', err.message); + } + + const resolveAttributeId = async (rawName) => { + const normalized = normalizeAttributeName(rawName); + const key = normalized.toLowerCase(); + if (attributeMap.has(key)) { + return attributeMap.get(key); + } + try { + const created = await store.attributename.create(googleAuthId, { + name: normalized, + exclusive: true, + }); + attributeMap.set(key, created.id); + return created.id; + } catch (err) { + log.warn(`Could not create attribute "${normalized}":`, err.message); + return null; + } + }; + + // Group children by parent SKU + const parentChildrenMap = new Map(); + for (const child of includedArticles) { + const parentJtlId = String(child.kVaterArtikel || '').trim(); + if (!parentJtlId || parentJtlId === '0') continue; + + const parentSku = jtlArtIdToSku.get(parentJtlId); + const childSku = cleanValue(child.cArtNr); + if (!parentSku || !childSku || parentSku === childSku) continue; + + if (!parentChildrenMap.has(parentSku)) { + const parentRow = includedArticles.find((a) => String(a.kArtikel) === parentJtlId) || null; + parentChildrenMap.set(parentSku, { parentRow, children: [] }); + } + parentChildrenMap.get(parentSku).children.push(child); + } + + let variantsLinked = 0; + for (const [parentSku, { parentRow, children }] of parentChildrenMap.entries()) { + const rawAttrName = parentRow ? cleanValue(parentRow.kEigenschaftKombi) : ''; + const targetAttrId = await resolveAttributeId(rawAttrName || 'Variante'); + + if (targetAttrId) { + try { + await store.article.toggleParent(googleAuthId, parentSku, { + isParent: true, + attributeNameId: targetAttrId, + }); + } catch (err) { + log.warn(`Could not toggle parent for ${parentSku}:`, err.message); + } + } + + const usedValues = new Set(); + for (const child of children) { + const childSku = cleanValue(child.cArtNr); + let val = cleanValue(child.kEigenschaftKombi) || 'Standard'; + + if (usedValues.has(val.toLowerCase())) { + val = `${val} (${childSku})`; + } + usedValues.add(val.toLowerCase()); + + try { + if (targetAttrId) { + await store.article.attachChild(googleAuthId, parentSku, { + sku: childSku, + value: val, + reparent: true, + }); + variantsLinked++; + } else { + await store.article.setParentBySku(googleAuthId, childSku, parentSku); + variantsLinked++; + } + } catch (err) { + log.warn(`Failed to attach child ${childSku} to parent ${parentSku}:`, err.message); + } + } + } + + log.info(`Linked ${variantsLinked} variants across ${parentChildrenMap.size} parents`); + + return { + variantsLinked, + parentsCount: parentChildrenMap.size, + }; +} + +// #endregion