dashback/src/services/companyImport.js

// MARK: - Imports
// #region Imports
import fs from 'fs';
import path from 'path';
import { getStore } from '../db/store.js';
import { createLogger } from '../util/logger.js';

import { cleanValue, parseGermanFloat, parseCsvRecords } from './companyImport/csvParser.js';
import { computeSkeleHash } from './companyImport/imageHasher.js';
import { normalizeAttributeName, extractVariantValue } from './companyImport/variantHelpers.js';
import { createProgressTracker } from './companyImport/progressTracker.js';
import { filterCatalogRecords, prepareImportTasks } from './companyImport/catalogPreprocessor.js';
import { importCategories } from './companyImport/categoryImporter.js';
import { importBrands } from './companyImport/brandImporter.js';
import { importArticles } from './companyImport/articleImporter.js';
import { importVariants } from './companyImport/variantImporter.js';
import { importArticleImages } from './companyImport/articleImageImporter.js';
import { importOrders } from './companyImport/orderImporter.js';
// #endregion

// MARK: - Re-exports
// #region Re-exports
export {
    cleanValue,
    parseGermanFloat,
    parseCsvRecords,
    computeSkeleHash,
    normalizeAttributeName,
    extractVariantValue,
};
// #endregion

const log = createLogger('import');

// MARK: - Company Import Orchestrator
// #region Company Import Orchestrator

/**
 * Executes full catalog import from preserved files in importDir.
 *
 * @param {object} opts
 * @param {number} opts.googleAuthId
 * @param {number} opts.companyId
 * @param {string} opts.importDir
 * @param {function} [opts.onProgress] - (counts) => void; throttled to 200ms
 * @param {function} [opts.onImages] - (skelehashes: string[]) => void; batched every 50ms
 * @returns {Promise<object>} Summary outcome with statistics
 */
export async function executeCompanyImport({
    googleAuthId,
    companyId,
    importDir,
    onProgress = () => {},
    onImages = () => {},
}) {
    const store = await getStore();

    // MARK: - Verify and Read Input Files
    // #region Read CSVs
    const catCsvPath = path.join(importDir, 'categories.csv');
    const artCsvPath = path.join(importDir, 'articles.csv');

    if (!fs.existsSync(catCsvPath) || !fs.existsSync(artCsvPath)) {
        throw new Error('Import files missing: categories.csv and articles.csv are required');
    }

    const catRecords = parseCsvRecords(fs.readFileSync(catCsvPath, 'utf8'));
    const artRecords = parseCsvRecords(fs.readFileSync(artCsvPath, 'utf8'));
    log.info(`Read ${catRecords.length} category rows and ${artRecords.length} article rows`);
    // #endregion

    // MARK: - Filter Records & Prepare Tasks
    // #region Preprocessing & Task Discovery
    const { includedCategories, includedArticles } = filterCatalogRecords(catRecords, artRecords);

    const {
        categoryImageTasks,
        brandMap,
        brandImageTasks,
        articleImageTasks,
        totalImagesCount,
        manRecords,
        itemRecords,
        totalReceiptsCount,
    } = prepareImportTasks({ importDir, includedCategories, includedArticles });
    // #endregion

    // MARK: - Progress Tracker Initialization
    // #region Progress Setup
    const progressTracker = createProgressTracker({
        initialCounts: {
            categoriestotal: includedCategories.length,
            categoriesdone: 0,
            articlestotal: includedArticles.length,
            articlesdone: 0,
            imagestotal: totalImagesCount,
            imagesdone: 0,
            receiptstotal: totalReceiptsCount,
            receiptsdone: 0,
        },
        onProgress,
        onImages,
    });

    // Emit baseline counts immediately
    progressTracker.emitProgress(true);
    // #endregion

    // MARK: - Phase 1: Categories
    // #region Phase 1: Categories
    const categoriesResult = await importCategories({
        store,
        googleAuthId,
        includedCategories,
        categoryImageTasks,
        progressTracker,
    });
    // #endregion

    // MARK: - Phase 2: Brands
    // #region Phase 2: Brands
    const brandsResult = await importBrands({
        store,
        googleAuthId,
        brandMap,
        brandImageTasks,
        progressTracker,
    });
    // #endregion

    // MARK: - Phase 3: Articles
    // #region Phase 3: Articles
    const articlesResult = await importArticles({
        store,
        googleAuthId,
        includedArticles,
        jtlCatIdToDashCatId: categoriesResult.jtlCatIdToDashCatId,
        hauptDashId: categoriesResult.hauptDashId,
        brandNameToId: brandsResult.brandNameToId,
        progressTracker,
    });
    // #endregion

    // MARK: - Phase 4: Variants
    // #region Phase 4: Variants
    const variantsResult = await importVariants({
        store,
        googleAuthId,
        includedArticles,
        jtlArtIdToSku: articlesResult.jtlArtIdToSku,
    });
    // #endregion

    // MARK: - Phase 5: Article Pictures
    // #region Phase 5: Article Pictures
    const imagesResult = await importArticleImages({
        store,
        googleAuthId,
        articleImageTasks,
        progressTracker,
    });
    // #endregion

    // MARK: - Phase 6: Orders / Receipts
    // #region Phase 6: Orders / Receipts
    const ordersResult = await importOrders({
        store,
        googleAuthId,
        manRecords,
        itemRecords,
        progressTracker,
    });
    // #endregion

    // Flush any pending images and progress updates
    progressTracker.flushSkeleHashes();
    progressTracker.emitProgress(true);

    return {
        ok: true,
        message: 'Import erfolgreich abgeschlossen.',
        stats: {
            categories: categoriesResult.jtlCatIdToDashCatId.size,
            categoriesCreated: categoriesResult.createdCatCount,
            categoryPictures: categoriesResult.catPicCount,
            brands: brandsResult.brandNameToId.size,
            brandsCreated: brandsResult.createdBrandCount,
            brandPictures: brandsResult.brandPicCount,
            articlesTotal: includedArticles.length,
            articlesImported: articlesResult.articlesImported,
            articlesUpdated: articlesResult.articlesUpdated,
            variantsLinked: variantsResult.variantsLinked,
            articlePictures: imagesResult.totalImagesImported,
            ordersTotal: totalReceiptsCount,
            ordersImported: ordersResult.ordersImported,
            ordersSkipped: ordersResult.ordersSkipped,
        },
    };
}

// #endregion