This commit is contained in:
seb
2026-10-11 18:20:35 +02:00
parent 1eb5cb4e59
commit fe7be79b90
12 changed files with 1395 additions and 937 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,55 @@
// MARK: - Imports
// #region Imports
import fs from 'fs';
import { computeSkeleHash } from './imageHasher.js';
import { createLogger } from '../../util/logger.js';
// #endregion
const log = createLogger('import');
// MARK: - Article Image Importer
// #region Article Image Importer
/**
* Imports article images and emits their skelehashes.
*
* @param {object} params
* @param {object} params.store - Store singleton
* @param {number} params.googleAuthId
* @param {Array<object>} params.articleImageTasks
* @param {object} params.progressTracker
* @returns {Promise<{ totalImagesImported: number }>}
*/
export async function importArticleImages({
store,
googleAuthId,
articleImageTasks,
progressTracker,
}) {
let totalImagesImported = 0;
for (const task of articleImageTasks) {
try {
const imgBuf = fs.readFileSync(task.fullPath);
await store.article.addImage(googleAuthId, task.sku, imgBuf);
totalImagesImported++;
const skele = await computeSkeleHash(imgBuf);
progressTracker.emitSkeleHash(skele?.base62 || null);
} catch (err) {
log.warn(`Article image add failed for ${task.sku} (${task.relPath}):`, err.message);
}
progressTracker.counts.imagesdone++;
progressTracker.emitProgress();
}
progressTracker.flushSkeleHashes();
progressTracker.emitProgress(true);
log.info(`Article pictures complete: ${totalImagesImported}/${articleImageTasks.length}`);
return {
totalImagesImported,
};
}
// #endregion

View File

@@ -0,0 +1,190 @@
// MARK: - Imports
// #region Imports
import { cleanValue, parseGermanFloat } from './csvParser.js';
import { createLogger } from '../../util/logger.js';
// #endregion
const log = createLogger('import');
// MARK: - Article Importer
// #region Article Importer
/**
* Imports articles (insert or update), setting prices, categories, barcodes, and brands.
*
* @param {object} params
* @param {object} params.store - Store singleton
* @param {number} params.googleAuthId
* @param {Array<Record<string, string>>} params.includedArticles
* @param {Map<string, number>} params.jtlCatIdToDashCatId
* @param {number} params.hauptDashId
* @param {Map<string, number>} params.brandNameToId
* @param {object} params.progressTracker
* @returns {Promise<{ jtlArtIdToSku: Map<string, string>, articlesImported: number, articlesUpdated: number }>}
*/
export async function importArticles({
store,
googleAuthId,
includedArticles,
jtlCatIdToDashCatId,
hauptDashId,
brandNameToId,
progressTracker,
}) {
const jtlArtIdToSku = new Map();
let articlesImported = 0;
let articlesUpdated = 0;
for (const a of includedArticles) {
const jtlArtId = String(a.kArtikel);
const sku = cleanValue(a.cArtNr);
if (!sku) {
progressTracker.counts.articlesdone++;
continue;
}
jtlArtIdToSku.set(jtlArtId, sku);
let name = cleanValue(a.cName);
let desc = cleanValue(a.descriptions);
if (!name && desc) {
try {
const parsed = JSON.parse(desc);
const item = Array.isArray(parsed) ? parsed[0] : parsed;
if (item?.cName) name = String(item.cName).trim();
if (item?.cBeschreibung) desc = String(item.cBeschreibung).trim();
} catch {}
}
if (!name) name = `Artikel ${sku}`;
// Category mapping
const assignedDashCatIds = [];
const rawCatStr = String(a.categories || '').trim();
if (rawCatStr) {
let parsedIds = [];
if (rawCatStr.startsWith('[') || rawCatStr.startsWith('{')) {
try {
const raw = JSON.parse(rawCatStr);
parsedIds = (Array.isArray(raw) ? raw : [raw]).map((it) => String(it.kKategorie || it));
} catch {
parsedIds = rawCatStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean);
}
} else {
parsedIds = rawCatStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean);
}
for (const jtlCId of parsedIds) {
if (jtlCatIdToDashCatId.has(jtlCId)) {
assignedDashCatIds.push(jtlCatIdToDashCatId.get(jtlCId));
}
}
}
const primaryCatId = assignedDashCatIds[0] || hauptDashId;
const trackStock = a.cLagerAktiv === 'Y' ? 1 : 0;
const netPrice = parseGermanFloat(a.fVKNetto, 0);
const taxRate = a.kSteuerklasse === '3' ? 7 : 19;
const barcode = cleanValue(a.cBarcode);
const hersteller = cleanValue(a.Hersteller);
const brandId = hersteller ? (brandNameToId.get(hersteller.toLowerCase()) || null) : null;
let existingArticle = null;
try {
const res = await store.article.getBySku(googleAuthId, sku);
existingArticle = res.article;
} catch {
existingArticle = null;
}
try {
if (existingArticle) {
await store.article.updateBySku(googleAuthId, sku, {
name,
track_stock: trackStock,
desc: desc || undefined,
});
await store.article.setPricesBySku(googleAuthId, sku, [
{ net: netPrice, tax: taxRate },
]);
if (assignedDashCatIds.length > 0) {
await store.article.setCategoryIdsBySku(googleAuthId, sku, assignedDashCatIds);
}
if (barcode) {
try {
const currentEans = existingArticle.eans || [];
if (!currentEans.includes(barcode)) {
await store.article.setEansBySku(googleAuthId, sku, [...currentEans, barcode]);
}
} catch (eanErr) {
log.warn(`Could not set EAN for ${sku}:`, eanErr.message);
}
}
if (brandId && existingArticle.brand?.id !== brandId) {
try {
await store.article.setBrandIdBySku(googleAuthId, sku, brandId);
} catch (bErr) {
log.warn(`Could not set brand for ${sku}:`, bErr.message);
}
}
articlesUpdated++;
} else {
await store.article.createUnderParent({
googleAuthId,
parentId: primaryCatId,
name,
sku,
track_stock: trackStock,
desc: desc || undefined,
});
await store.article.setPricesBySku(googleAuthId, sku, [
{ net: netPrice, tax: taxRate },
]);
if (assignedDashCatIds.length > 1) {
await store.article.setCategoryIdsBySku(googleAuthId, sku, assignedDashCatIds);
}
if (barcode) {
try {
await store.article.setEansBySku(googleAuthId, sku, [barcode]);
} catch (eanErr) {
log.warn(`Could not set EAN for ${sku}:`, eanErr.message);
}
}
if (brandId) {
try {
await store.article.setBrandIdBySku(googleAuthId, sku, brandId);
} catch (bErr) {
log.warn(`Could not set brand for ${sku}:`, bErr.message);
}
}
articlesImported++;
}
} catch (err) {
log.warn(`Failed to save article ${sku} ("${name}"):`, err.message);
}
progressTracker.counts.articlesdone++;
progressTracker.emitProgress();
}
progressTracker.emitProgress(true);
log.info(`Articles complete: New: ${articlesImported}, Updated: ${articlesUpdated}`);
return {
jtlArtIdToSku,
articlesImported,
articlesUpdated,
};
}
// #endregion

View File

@@ -0,0 +1,91 @@
// MARK: - Imports
// #region Imports
import fs from 'fs';
import { computeSkeleHash } from './imageHasher.js';
import { createLogger } from '../../util/logger.js';
// #endregion
const log = createLogger('import');
// MARK: - Brand Importer
// #region Brand Importer
/**
* Imports brand manufacturers and attaches brand logos.
*
* @param {object} params
* @param {object} params.store - Store singleton
* @param {number} params.googleAuthId
* @param {Map<string, Array<{ relPath: string, fullPath: string }>>} params.brandMap
* @param {Array<object>} params.brandImageTasks
* @param {object} params.progressTracker
* @returns {Promise<{ brandNameToId: Map<string, number>, createdBrandCount: number, brandPicCount: number }>}
*/
export async function importBrands({
store,
googleAuthId,
brandMap,
brandImageTasks,
progressTracker,
}) {
const brandNameToId = new Map();
let createdBrandCount = 0;
let brandPicCount = 0;
const { brands: existingBrandRows } = await store.brand.list(googleAuthId, { offset: 0, limit: 100 });
const existingBrandNames = new Set(
existingBrandRows.map((b) => (b.name || '').trim().toLowerCase()).filter(Boolean)
);
const brandTaskMap = new Map();
for (const task of brandImageTasks) {
brandTaskMap.set(task.brandName.toLowerCase(), task);
}
for (const [brandName] of brandMap.entries()) {
const lowerName = brandName.toLowerCase();
const isNew = !existingBrandNames.has(lowerName);
try {
const brand = await store.brand.upsert(googleAuthId, { name: brandName });
brandNameToId.set(lowerName, brand.id);
if (isNew) {
createdBrandCount++;
existingBrandNames.add(lowerName);
}
const imgTask = brandTaskMap.get(lowerName);
if (imgTask) {
if (!brand.pics || brand.pics.length === 0) {
try {
const imgBuf = fs.readFileSync(imgTask.fullPath);
await store.brand.addImage(googleAuthId, brand.id, imgBuf);
brandPicCount++;
const skele = await computeSkeleHash(imgBuf);
progressTracker.emitSkeleHash(skele?.base62 || null);
} catch (err) {
log.warn(`Brand image error for "${brandName}":`, err.message);
}
}
progressTracker.counts.imagesdone++;
}
} catch (err) {
log.warn(`Failed to upsert brand "${brandName}":`, err.message);
}
progressTracker.emitProgress();
}
progressTracker.flushSkeleHashes();
progressTracker.emitProgress(true);
log.info(`Brands resolved: ${brandNameToId.size}, created: ${createdBrandCount}, images: ${brandPicCount}`);
return {
brandNameToId,
createdBrandCount,
brandPicCount,
};
}
// #endregion

View File

@@ -0,0 +1,170 @@
// MARK: - Imports
// #region Imports
import fs from 'fs';
import path from 'path';
import { cleanValue, parseCsvRecords } from './csvParser.js';
// #endregion
// MARK: - Category & Article Filtering
// #region Category & Article Filtering
/**
* Filters out trash/papierkorb categories and their descendants,
* as well as articles that exclusively belong to excluded categories.
*
* @param {Array<Record<string, string>>} catRecords
* @param {Array<Record<string, string>>} artRecords
* @returns {{ includedCategories: Array<Record<string, string>>, includedArticles: Array<Record<string, string>>, excludedJtlIds: Set<string> }}
*/
export function filterCatalogRecords(catRecords, artRecords) {
const excludedJtlIds = new Set(['72']);
for (const c of catRecords) {
let name = (c.cName || '').trim();
if (!name) {
try {
const tr = JSON.parse(c.translations || '[]');
name = tr[0]?.cName || '';
} catch {}
}
if (name.toLowerCase().includes('papierkorb') || name.toLowerCase().includes('trash')) {
excludedJtlIds.add(String(c.kKategorie));
}
}
let addedExcluded = true;
while (addedExcluded) {
addedExcluded = false;
for (const c of catRecords) {
const id = String(c.kKategorie);
const parent = String(c.kOberKategorie);
if (!excludedJtlIds.has(id) && excludedJtlIds.has(parent)) {
excludedJtlIds.add(id);
addedExcluded = true;
}
}
}
const includedCategories = catRecords.filter((c) => !excludedJtlIds.has(String(c.kKategorie)));
const includedArticles = artRecords.filter((a) => {
const catStr = String(a.categories || '').trim();
if (!catStr) return true;
let catIds = [];
if (catStr.startsWith('[') || catStr.startsWith('{')) {
try {
const raw = JSON.parse(catStr);
catIds = (Array.isArray(raw) ? raw : [raw]).map((it) => String(it.kKategorie || it));
} catch {
catIds = catStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean);
}
} else {
catIds = catStr.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean);
}
if (catIds.length > 0 && catIds.every((cid) => excludedJtlIds.has(cid))) {
return false;
}
return true;
});
return { includedCategories, includedArticles, excludedJtlIds };
}
// #endregion
// MARK: - Task Collection & Manifests
// #region Task Collection & Manifests
/**
* Scans directories for existing category, brand, and article images,
* and loads receipts / manifest records if available.
*
* @param {object} params
* @param {string} params.importDir
* @param {Array<Record<string, string>>} params.includedCategories
* @param {Array<Record<string, string>>} params.includedArticles
* @returns {object} Discovered image tasks, brand map, and manifest records
*/
export function prepareImportTasks({ importDir, includedCategories, includedArticles }) {
// Category images
const categoryImageTasks = [];
for (const c of includedCategories) {
const picRel = (c.pictureFiles || '').trim();
if (picRel) {
for (const rel of picRel.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean)) {
const full = path.resolve(importDir, rel);
if (fs.existsSync(full)) {
categoryImageTasks.push({ catJtlId: String(c.kKategorie), relPath: rel, fullPath: full });
}
}
}
}
// Brand images (unique manufacturers, 1 primary logo)
const brandMap = new Map(); // brandName -> array of { relPath, fullPath }
for (const a of includedArticles) {
const brand = cleanValue(a.Hersteller);
if (!brand) continue;
if (!brandMap.has(brand)) {
brandMap.set(brand, []);
const rawPics = String(a.manufacturerPictureFiles || '').trim();
if (rawPics) {
for (const rel of rawPics.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean)) {
const full = path.resolve(importDir, rel);
if (fs.existsSync(full)) {
brandMap.get(brand).push({ relPath: rel, fullPath: full });
}
}
}
}
}
const brandImageTasks = [];
for (const [brandName, pics] of brandMap.entries()) {
if (pics.length > 0) {
brandImageTasks.push({ brandName, ...pics[0] });
}
}
// Article images
const articleImageTasks = [];
for (const a of includedArticles) {
const sku = cleanValue(a.cArtNr);
if (!sku) continue;
const rawPics = String(a.pictureFiles || '').trim();
if (rawPics) {
for (const rel of rawPics.split(/[\r\n,;]+/).map((s) => s.trim()).filter(Boolean)) {
const full = path.resolve(importDir, rel);
if (fs.existsSync(full)) {
articleImageTasks.push({ sku, relPath: rel, fullPath: full });
}
}
}
}
const totalImagesCount = categoryImageTasks.length + brandImageTasks.length + articleImageTasks.length;
// Receipts / Manifests
const manCsvPath = path.join(importDir, 'manifests.csv');
const itemsCsvPath = path.join(importDir, 'manifestitems.csv');
let manRecords = [];
let itemRecords = [];
if (fs.existsSync(manCsvPath) && fs.existsSync(itemsCsvPath)) {
manRecords = parseCsvRecords(fs.readFileSync(manCsvPath, 'utf8'));
itemRecords = parseCsvRecords(fs.readFileSync(itemsCsvPath, 'utf8'));
}
const totalReceiptsCount = manRecords.length;
return {
categoryImageTasks,
brandMap,
brandImageTasks,
articleImageTasks,
totalImagesCount,
manRecords,
itemRecords,
totalReceiptsCount,
};
}
// #endregion

View File

@@ -0,0 +1,148 @@
// MARK: - Imports
// #region Imports
import fs from 'fs';
import { cleanValue } from './csvParser.js';
import { computeSkeleHash } from './imageHasher.js';
import { createLogger } from '../../util/logger.js';
// #endregion
const log = createLogger('import');
// MARK: - Category Importer
// #region Category Importer
/**
* Imports category tree and attaches category images.
*
* @param {object} params
* @param {object} params.store - Store singleton
* @param {number} params.googleAuthId
* @param {Array<Record<string, string>>} params.includedCategories
* @param {Array<object>} params.categoryImageTasks
* @param {object} params.progressTracker
* @returns {Promise<{ jtlCatIdToDashCatId: Map<string, number>, hauptDashId: number, createdCatCount: number, catPicCount: number }>}
*/
export async function importCategories({
store,
googleAuthId,
includedCategories,
categoryImageTasks,
progressTracker,
}) {
const treeResult = await store.category.getTreeForCompany(googleAuthId);
const existingTree = treeResult.tree || [];
const rootCategory = existingTree.find((c) => c.parentId == null) || existingTree[0];
if (!rootCategory) {
throw new Error('Company root category not found');
}
const rootCategoryId = rootCategory.id;
const existingMap = new Map();
for (const c of existingTree) {
const key = `${c.parentId ?? 'root'}:${String(c.name || '').trim().toLowerCase()}`;
existingMap.set(key, c.id);
}
const jtlCatIdToDashCatId = new Map();
jtlCatIdToDashCatId.set('0', rootCategoryId);
// Group category images by JTL category ID
const catImagesByJtlId = new Map();
for (const task of categoryImageTasks) {
if (!catImagesByJtlId.has(task.catJtlId)) {
catImagesByJtlId.set(task.catJtlId, []);
}
catImagesByJtlId.get(task.catJtlId).push(task);
}
const pendingCats = [...includedCategories];
let createdCatCount = 0;
let catPicCount = 0;
let safety = 0;
while (pendingCats.length > 0 && safety < 10) {
safety++;
const remaining = [];
for (const c of pendingCats) {
const jtlId = String(c.kKategorie);
const jtlParent = String(c.kOberKategorie || '0');
let name = cleanValue(c.cName);
if (!name) {
try {
const tr = JSON.parse(c.translations || '[]');
name = tr[0]?.cName || '';
} catch {}
}
if (!name) name = `Kategorie ${jtlId}`;
let dashParentId;
if (jtlParent === '0' || !jtlParent) {
dashParentId = rootCategoryId;
} else if (jtlCatIdToDashCatId.has(jtlParent)) {
dashParentId = jtlCatIdToDashCatId.get(jtlParent);
} else {
remaining.push(c);
continue;
}
const key = `${dashParentId}:${name.toLowerCase()}`;
let dashCatId = existingMap.get(key);
if (!dashCatId) {
try {
const created = await store.category.createUnderParent({
googleAuthId,
parentId: dashParentId,
name,
});
dashCatId = created.id;
existingMap.set(key, dashCatId);
createdCatCount++;
} catch (err) {
log.warn(`Failed to create category "${name}":`, err.message);
}
}
if (dashCatId) {
jtlCatIdToDashCatId.set(jtlId, dashCatId);
// Add category images
const imgs = catImagesByJtlId.get(jtlId) || [];
for (const imgTask of imgs) {
try {
const imgBuf = fs.readFileSync(imgTask.fullPath);
await store.category.addImage(googleAuthId, dashCatId, imgBuf);
catPicCount++;
const skele = await computeSkeleHash(imgBuf);
progressTracker.emitSkeleHash(skele?.base62 || null);
} catch (err) {
log.warn(`Category image error (${imgTask.relPath}):`, err.message);
}
progressTracker.counts.imagesdone++;
}
}
progressTracker.counts.categoriesdone++;
progressTracker.emitProgress();
}
pendingCats.length = 0;
pendingCats.push(...remaining);
}
progressTracker.flushSkeleHashes();
progressTracker.emitProgress(true);
log.info(`Categories resolved: ${jtlCatIdToDashCatId.size}/${includedCategories.length}, created: ${createdCatCount}, images: ${catPicCount}`);
const hauptDashId = jtlCatIdToDashCatId.get('1') || rootCategoryId;
return {
jtlCatIdToDashCatId,
hauptDashId,
createdCatCount,
catPicCount,
};
}
// #endregion

View File

@@ -0,0 +1,128 @@
// MARK: - CSV Parsing & Value Sanitization
// #region CSV Parsing & Value Sanitization
/**
* Cleans formula prefixes (="..."), quotes, and whitespace from CSV values.
*
* @param {string|any} str
* @returns {string}
*/
export function cleanValue(str) {
if (str == null) return '';
let val = String(str).trim();
if (val.startsWith('=')) {
val = val.slice(1).trim();
}
if (val.startsWith('"') && val.endsWith('"')) {
val = val.slice(1, -1).trim();
}
return val;
}
/**
* Parses German decimal format numbers (e.g. "8,36" -> 8.36).
*
* @param {string|number} str
* @param {number} [fallback=0]
* @returns {number}
*/
export function parseGermanFloat(str, fallback = 0) {
const clean = cleanValue(str).replace(',', '.');
const num = parseFloat(clean);
return Number.isNaN(num) ? fallback : num;
}
/**
* Parses CSV text into an array of objects keyed by header.
* Supports auto-detecting semicolon (;) vs comma (,) delimiters,
* quoted fields with embedded newlines/quotes, and formula prefixes.
*
* @param {string} text - Raw CSV text
* @returns {Array<Record<string, string>>}
*/
export function parseCsvRecords(text) {
const clean = text.replace(/^\uFEFF/, '');
const firstLine = clean.split(/[\r\n]/)[0] || '';
const semiCount = (firstLine.match(/;/g) || []).length;
const commaCount = (firstLine.match(/,/g) || []).length;
const delimiter = semiCount >= commaCount ? ';' : ',';
const rows = [];
let field = '';
let row = [];
let inQuotes = false;
let i = 0;
const len = clean.length;
while (i < len) {
const c = clean[i];
if (inQuotes) {
if (c === '"') {
if (i + 1 < len && clean[i + 1] === '"') {
field += '"';
i += 2;
continue;
} else {
inQuotes = false;
i++;
continue;
}
} else {
field += c;
i++;
continue;
}
} else {
if (c === '"') {
inQuotes = true;
i++;
continue;
} else if (c === delimiter) {
row.push(cleanValue(field));
field = '';
i++;
continue;
} else if (c === '\r') {
if (i + 1 < len && clean[i + 1] === '\n') {
i++;
}
row.push(cleanValue(field));
rows.push(row);
field = '';
row = [];
i++;
continue;
} else if (c === '\n') {
row.push(cleanValue(field));
rows.push(row);
field = '';
row = [];
i++;
continue;
} else {
field += c;
i++;
continue;
}
}
}
if (field !== '' || row.length > 0) {
row.push(cleanValue(field));
rows.push(row);
}
if (rows.length === 0) return [];
const header = rows[0].map((h) => cleanValue(h));
const records = [];
for (let r = 1; r < rows.length; r++) {
const curr = rows[r];
if (curr.length === 1 && curr[0] === '') continue;
const obj = {};
for (let col = 0; col < header.length; col++) {
obj[header[col]] = curr[col] ?? '';
}
records.push(obj);
}
return records;
}
// #endregion

View File

@@ -0,0 +1,41 @@
// MARK: - Imports
// #region Imports
import sharp from 'sharp';
import { encodeRGBA } from 'skelehash-wasm';
import { createLogger } from '../../util/logger.js';
// #endregion
const log = createLogger('import');
// MARK: - SkeleHash Computation
// #region SkeleHash Computation
/**
* Computes SkeleHash base62 hash for an image buffer.
*
* @param {Buffer} buffer
* @returns {Promise<{base62: string, width: number, height: number, hashLength: number}|null>}
*/
export async function computeSkeleHash(buffer) {
try {
const { data, info } = await sharp(buffer)
.rotate()
.resize({ width: 200, height: 200, fit: 'inside' })
.ensureAlpha()
.raw()
.toBuffer({ resolveWithObject: true });
const result = await encodeRGBA(info.width, info.height, data);
return {
base62: result.base62,
width: info.width,
height: info.height,
hashLength: result.hash?.length ?? 0,
};
} catch (err) {
log.warn('Failed to compute skelehash for image:', err.message);
return null;
}
}
// #endregion

View File

@@ -0,0 +1,165 @@
// MARK: - Imports
// #region Imports
import { cleanValue, parseGermanFloat } from './csvParser.js';
import { createLogger } from '../../util/logger.js';
// #endregion
const log = createLogger('import');
// MARK: - Order / Manifest Importer
// #region Order / Manifest Importer
/**
* Imports order manifests and line items, resolving customer entities.
*
* @param {object} params
* @param {object} params.store - Store singleton
* @param {number} params.googleAuthId
* @param {Array<Record<string, string>>} params.manRecords
* @param {Array<Record<string, string>>} params.itemRecords
* @param {object} params.progressTracker
* @returns {Promise<{ ordersImported: number, ordersSkipped: number }>}
*/
export async function importOrders({
store,
googleAuthId,
manRecords,
itemRecords,
progressTracker,
}) {
let ordersImported = 0;
let ordersSkipped = 0;
if (manRecords.length === 0 || itemRecords.length === 0) {
return { ordersImported, ordersSkipped };
}
const itemsByAuftrag = new Map();
for (const it of itemRecords) {
const kAuftrag = String(it.kAuftrag || '').trim();
if (!kAuftrag) continue;
if (!itemsByAuftrag.has(kAuftrag)) {
itemsByAuftrag.set(kAuftrag, []);
}
itemsByAuftrag.get(kAuftrag).push(it);
}
const existingOrdersResult = await store.order.list(googleAuthId);
const existingOrderList = Array.isArray(existingOrdersResult)
? existingOrdersResult
: (existingOrdersResult?.orders || []);
const existingOrderNumbers = new Set(
existingOrderList.map((o) => (o.number || '').trim()).filter(Boolean)
);
let defaultCustomerGroupId = null;
try {
const groups = await store.customergroup.list(googleAuthId);
defaultCustomerGroupId = groups[0]?.id || null;
} catch {}
const entityMap = new Map();
try {
const entities = await store.entity.list(googleAuthId);
for (const ent of entities) {
if (ent.name) {
entityMap.set(ent.name.trim().toLowerCase(), ent.id);
}
}
} catch {}
const resolveEntityId = async (name, firstName) => {
const cleanName = (name || '').trim();
if (!cleanName || !defaultCustomerGroupId) return null;
const key = cleanName.toLowerCase();
if (entityMap.has(key)) return entityMap.get(key);
try {
const ent = await store.entity.create(googleAuthId, {
customergroupId: defaultCustomerGroupId,
name: cleanName,
firstName: (firstName || '').trim() || undefined,
});
entityMap.set(key, ent.id);
return ent.id;
} catch (err) {
log.warn(`Could not create entity "${cleanName}":`, err.message);
return null;
}
};
for (const m of manRecords) {
const orderNumber = cleanValue(m.cAuftragsNr);
if (!orderNumber || existingOrderNumbers.has(orderNumber)) {
ordersSkipped++;
progressTracker.counts.receiptsdone++;
progressTracker.emitProgress();
continue;
}
const rawItems = itemsByAuftrag.get(String(m.kAuftrag || '').trim()) || [];
if (rawItems.length === 0) {
ordersSkipped++;
progressTracker.counts.receiptsdone++;
progressTracker.emitProgress();
continue;
}
const lines = rawItems.map((it) => {
const sku = cleanValue(it.cArtNr) || null;
const rawName = cleanValue(it.cName || it.cNameStandard);
const name = rawName || (sku ? `Artikel ${sku}` : 'Artikel');
const amount = parseGermanFloat(it.fAnzahl, 1);
const net = parseGermanFloat(it.fVKNetto, 0);
const taxrate = parseGermanFloat(it.fMwSt, 19);
const rebate = parseGermanFloat(it.fRabatt, 0);
const comment = cleanValue(it.cHinweis) || null;
return {
sku,
name,
amount,
net,
taxrate,
rebate,
comment,
};
});
const custName = cleanValue(m.cName) || 'Laufkunde';
const custFirstName = cleanValue(m.cVorname);
const entityId = await resolveEntityId(custName, custFirstName);
const dateStr = cleanValue(m.dErstellt);
const parsedDate = dateStr ? new Date(dateStr) : new Date();
const validDate = !Number.isNaN(parsedDate.getTime()) ? parsedDate.toISOString() : new Date().toISOString();
try {
await store.order.create(googleAuthId, {
number: orderNumber,
externalNumber: cleanValue(m.cExterneAuftragsnummer) || null,
date: validDate,
createdAt: validDate,
entityId: entityId || undefined,
lines,
});
existingOrderNumbers.add(orderNumber);
ordersImported++;
} catch (err) {
log.warn(`Failed to create order ${orderNumber}:`, err.message);
}
progressTracker.counts.receiptsdone++;
progressTracker.emitProgress();
}
progressTracker.emitProgress(true);
log.info(`Orders complete: ${ordersImported} created, ${ordersSkipped} skipped`);
return {
ordersImported,
ordersSkipped,
};
}
// #endregion

View File

@@ -0,0 +1,74 @@
// MARK: - Progress & Image Stream Tracking
// #region Progress & Image Stream Tracking
/**
* Creates an event tracking controller for throttled progress updates and batched skelehashes.
*
* @param {object} params
* @param {object} params.initialCounts - Initial totals and completed counts
* @param {function} [params.onProgress] - Throttled callback for progress counts
* @param {function} [params.onImages] - Batched callback for emitted skelehash base62 strings
* @param {number} [params.progressThrottleMs=200]
* @param {number} [params.imageBatchMs=50]
* @returns {object} Tracker with mutateable counts and emission methods
*/
export function createProgressTracker({
initialCounts,
onProgress = () => {},
onImages = () => {},
progressThrottleMs = 200,
imageBatchMs = 50,
}) {
const counts = {
categoriestotal: 0,
categoriesdone: 0,
articlestotal: 0,
articlesdone: 0,
imagestotal: 0,
imagesdone: 0,
receiptstotal: 0,
receiptsdone: 0,
...initialCounts,
};
let lastProgressEmit = 0;
const emitProgress = (force = false) => {
const now = Date.now();
if (force || now - lastProgressEmit >= progressThrottleMs) {
lastProgressEmit = now;
onProgress({ ...counts });
}
};
let skeleBuffer = [];
let lastSkeleEmit = Date.now();
const emitSkeleHash = (base62, force = false) => {
if (base62) skeleBuffer.push(base62);
const now = Date.now();
if (skeleBuffer.length > 0 && (force || now - lastSkeleEmit >= imageBatchMs)) {
const batch = skeleBuffer;
skeleBuffer = [];
lastSkeleEmit = now;
onImages(batch);
}
};
const flushSkeleHashes = () => {
if (skeleBuffer.length > 0) {
const batch = skeleBuffer;
skeleBuffer = [];
lastSkeleEmit = Date.now();
onImages(batch);
}
};
return {
counts,
emitProgress,
emitSkeleHash,
flushSkeleHashes,
};
}
// #endregion

View File

@@ -0,0 +1,53 @@
// MARK: - Variant Attribute & Value Helpers
// #region Variant Attribute & Value Helpers
/**
* Normalizes common variant attribute names and typos.
*
* @param {string} name
* @returns {string}
*/
export function normalizeAttributeName(name) {
const clean = (name || '').trim();
if (!clean) return 'Variante';
const lower = clean.toLowerCase();
if (lower === 'menge' || lower === 'mange' || lower === 'meneg') return 'Menge';
if (lower === 'farbe') return 'Farbe';
if (lower === 'größe' || lower === 'grosse') return 'Größe';
if (lower === 'sorte') return 'Sorte';
if (lower === 'geschmack') return 'Geschmack';
if (lower === 'typ') return 'Typ';
if (lower === 'stück' || lower === 'stueck') return 'Stück';
if (lower === 'ausführung' || lower === 'ausfuehrung') return 'Ausführung';
if (lower === 'motiv') return 'Motiv';
return clean.charAt(0).toUpperCase() + clean.slice(1);
}
/**
* Fallback variant value extraction if child has no explicit attribute value.
*
* @param {string} parentName
* @param {string} childName
* @param {string} childSku
* @param {string} parentSku
* @returns {string}
*/
export function extractVariantValue(parentName, childName, childSku, parentSku) {
let pName = (parentName || '').replace(/[©®™]/g, '').trim();
let cName = (childName || '').replace(/[©®™]/g, '').trim();
if (pName && cName) {
if (cName.toLowerCase().startsWith(pName.toLowerCase())) {
let val = cName.slice(pName.length).trim().replace(/^[-–—:/,\s]+/, '').trim();
if (val) return val;
}
}
if (parentSku && childSku && childSku.startsWith(parentSku)) {
let suffix = childSku.slice(parentSku.length).trim().replace(/^[-_]+/, '').trim();
if (suffix) return suffix;
}
if (cName && cName !== pName) return cName;
return childSku || 'Standard';
}
// #endregion

View File

@@ -0,0 +1,129 @@
// MARK: - Imports
// #region Imports
import { cleanValue } from './csvParser.js';
import { normalizeAttributeName } from './variantHelpers.js';
import { createLogger } from '../../util/logger.js';
// #endregion
const log = createLogger('import');
// MARK: - Variant Importer
// #region Variant Importer
/**
* Resolves attribute definitions and links variant child articles to parents.
*
* @param {object} params
* @param {object} params.store - Store singleton
* @param {number} params.googleAuthId
* @param {Array<Record<string, string>>} params.includedArticles
* @param {Map<string, string>} params.jtlArtIdToSku
* @returns {Promise<{ variantsLinked: number, parentsCount: number }>}
*/
export async function importVariants({
store,
googleAuthId,
includedArticles,
jtlArtIdToSku,
}) {
const attributeMap = new Map();
try {
const attrList = await store.attributename.list(googleAuthId);
for (const attr of attrList) {
if (attr.name) {
attributeMap.set(attr.name.trim().toLowerCase(), attr.id);
}
}
} catch (err) {
log.warn('Could not list attributes:', err.message);
}
const resolveAttributeId = async (rawName) => {
const normalized = normalizeAttributeName(rawName);
const key = normalized.toLowerCase();
if (attributeMap.has(key)) {
return attributeMap.get(key);
}
try {
const created = await store.attributename.create(googleAuthId, {
name: normalized,
exclusive: true,
});
attributeMap.set(key, created.id);
return created.id;
} catch (err) {
log.warn(`Could not create attribute "${normalized}":`, err.message);
return null;
}
};
// Group children by parent SKU
const parentChildrenMap = new Map();
for (const child of includedArticles) {
const parentJtlId = String(child.kVaterArtikel || '').trim();
if (!parentJtlId || parentJtlId === '0') continue;
const parentSku = jtlArtIdToSku.get(parentJtlId);
const childSku = cleanValue(child.cArtNr);
if (!parentSku || !childSku || parentSku === childSku) continue;
if (!parentChildrenMap.has(parentSku)) {
const parentRow = includedArticles.find((a) => String(a.kArtikel) === parentJtlId) || null;
parentChildrenMap.set(parentSku, { parentRow, children: [] });
}
parentChildrenMap.get(parentSku).children.push(child);
}
let variantsLinked = 0;
for (const [parentSku, { parentRow, children }] of parentChildrenMap.entries()) {
const rawAttrName = parentRow ? cleanValue(parentRow.kEigenschaftKombi) : '';
const targetAttrId = await resolveAttributeId(rawAttrName || 'Variante');
if (targetAttrId) {
try {
await store.article.toggleParent(googleAuthId, parentSku, {
isParent: true,
attributeNameId: targetAttrId,
});
} catch (err) {
log.warn(`Could not toggle parent for ${parentSku}:`, err.message);
}
}
const usedValues = new Set();
for (const child of children) {
const childSku = cleanValue(child.cArtNr);
let val = cleanValue(child.kEigenschaftKombi) || 'Standard';
if (usedValues.has(val.toLowerCase())) {
val = `${val} (${childSku})`;
}
usedValues.add(val.toLowerCase());
try {
if (targetAttrId) {
await store.article.attachChild(googleAuthId, parentSku, {
sku: childSku,
value: val,
reparent: true,
});
variantsLinked++;
} else {
await store.article.setParentBySku(googleAuthId, childSku, parentSku);
variantsLinked++;
}
} catch (err) {
log.warn(`Failed to attach child ${childSku} to parent ${parentSku}:`, err.message);
}
}
}
log.info(`Linked ${variantsLinked} variants across ${parentChildrenMap.size} parents`);
return {
variantsLinked,
parentsCount: parentChildrenMap.size,
};
}
// #endregion