feat: Enhance product management by adding WebSocket support for real-time updates, implement product loading in categories, and improve caching for category products.

2025-11-23 11:40:42 +01:00
parent 49246169db
commit 71910f84a2
4 changed files with 361 additions and 7 deletions
--- a/category-products-syncer.js
+++ b/category-products-syncer.js
@@ -0,0 +1,174 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { EventEmitter } from 'events';
+import { createConnection } from './database.js';
+
+class CategoryProductsSyncer extends EventEmitter {
+    constructor() {
+        super();
+        if (CategoryProductsSyncer.instance) {
+            return CategoryProductsSyncer.instance;
+        }
+        this.cacheBaseDir = process.env.CACHE_LOCATION || '.';
+        
+        // Track syncing state
+        this.isSyncing = false;
+        this.queuedCategoryIds = null;
+
+        CategoryProductsSyncer.instance = this;
+    }
+
+    async syncProducts(categoryIds) {
+        // Check if already syncing
+        if (this.isSyncing) {
+            console.log('⏳ CategoryProductsSyncer is busy. Queuing sync...');
+            this.queuedCategoryIds = categoryIds;
+            return;
+        }
+
+        this.isSyncing = true;
+        try {
+            await this._performSync(categoryIds);
+        } catch (err) {
+            console.error('❌ Error syncing products:', err);
+        } finally {
+            this.isSyncing = false;
+            // Process queued sync if exists
+            if (this.queuedCategoryIds) {
+                const nextIds = this.queuedCategoryIds;
+                this.queuedCategoryIds = null;
+                // Use setTimeout to allow event loop to breathe
+                setTimeout(() => this.syncProducts(nextIds), 0);
+            }
+        }
+    }
+
+    async _performSync(categoryIds) {
+        const startTime = Date.now();
+        const productsDir = path.join(this.cacheBaseDir, 'products');
+
+        // Ensure directory exists
+        await fs.mkdir(productsDir, { recursive: true });
+
+        // Get existing files
+        let existingFiles = [];
+        try {
+            existingFiles = await fs.readdir(productsDir);
+        } catch (err) {
+            // Directory might be empty or new
+        }
+
+        // Filter for category json files (assuming we save as category_{id}.json)
+        const existingIds = existingFiles
+            .filter(f => f.startsWith('category_') && f.endsWith('.json'))
+            .map(f => parseInt(f.replace('category_', '').replace('.json', '')));
+
+        const validIds = new Set(categoryIds.filter(id => id !== null && id !== undefined));
+
+        // 1. Delete obsolete category files
+        const toDelete = existingIds.filter(id => !validIds.has(id));
+        for (const id of toDelete) {
+            const filePath = path.join(productsDir, `category_${id}.json`);
+            await fs.unlink(filePath);
+        }
+        if (toDelete.length > 0) {
+            console.log(`🗑️  Deleted ${toDelete.length} obsolete product lists.`);
+        }
+
+        // 2. Update/Create product lists for all valid categories
+        // We update all because product assignments might have changed even if category exists
+        if (validIds.size > 0) {
+            console.log(`📦 Syncing products for ${validIds.size} categories...`);
+            await this._fetchAndWriteProducts([...validIds], productsDir);
+        } else {
+            console.log(`✅ No categories to sync products for.`);
+        }
+
+        const duration = Date.now() - startTime;
+        console.log(`✅ Product sync completed in ${duration}ms.`);
+    }
+
+    async _fetchAndWriteProducts(ids, dir) {
+        let pool;
+        try {
+            pool = await createConnection();
+
+            // Process in chunks to avoid huge queries
+            const chunkSize = 50;
+            for (let i = 0; i < ids.length; i += chunkSize) {
+                const chunk = ids.slice(i, i + chunkSize);
+                const list = chunk.join(',');
+
+                // Fetch products for this chunk of categories
+                // We need kArtikel and cName, ordered by bRowversion descending
+                const result = await pool.request().query(`
+                    SELECT 
+                        ka.kKategorie,
+                        ka.kArtikel,
+                        ab.cName
+                    FROM tkategorieartikel ka
+                    JOIN tArtikelBeschreibung ab ON ka.kArtikel = ab.kArtikel
+                    JOIN tArtikel a ON ka.kArtikel = a.kArtikel
+                    WHERE ab.kSprache = ${process.env.JTL_SPRACHE_ID}
+                      AND ab.kPlattform = ${process.env.JTL_PLATTFORM_ID}
+                      AND ab.kShop = ${process.env.JTL_SHOP_ID}
+                      AND ka.kKategorie IN (${list})
+                    ORDER BY a.bRowversion DESC, ab.bRowversion DESC
+                `);
+
+                // Group results by kKategorie
+                const productsByCategory = {};
+                
+                // Initialize arrays for all requested IDs (so we create empty files for empty categories)
+                chunk.forEach(id => {
+                    productsByCategory[id] = [];
+                });
+
+                for (const record of result.recordset) {
+                    if (productsByCategory[record.kKategorie]) {
+                        productsByCategory[record.kKategorie].push({
+                            kArtikel: record.kArtikel,
+                            cName: record.cName
+                        });
+                    }
+                }
+
+                // Write files
+                for (const catId of chunk) {
+                    const filePath = path.join(dir, `category_${catId}.json`);
+                    const products = productsByCategory[catId] || [];
+                    const newContent = JSON.stringify(products, null, 2);
+
+                    // Check for changes
+                    let oldContent = '';
+                    try {
+                        oldContent = await fs.readFile(filePath, 'utf-8');
+                    } catch (e) {
+                        // File doesn't exist yet
+                    }
+
+                    if (oldContent !== newContent) {
+                        await fs.writeFile(filePath, newContent);
+                        this.emit('categoryUpdated', { id: catId, products });
+                    }
+                }
+
+                const processed = Math.min(i + chunkSize, ids.length);
+                if (processed === ids.length) {
+                    console.log(`✅ Processed products for ${processed}/${ids.length} categories.`);
+                } else {
+                    console.log(`⏳ Processed products for ${processed}/${ids.length} categories...`);
+                }
+            }
+        } catch (err) {
+            console.error('❌ Error fetching products:', err);
+        } finally {
+            if (pool) {
+                await pool.close();
+            }
+        }
+    }
+}
+
+const instance = new CategoryProductsSyncer();
+export default instance;
--- a/index.html
+++ b/index.html
@@ -117,6 +117,21 @@
            border-radius: 8px;
            text-align: center;
        }
+        .category-products {
+            margin-top: 0.5rem;
+            padding-left: 4rem;
+            font-size: 0.9rem;
+            color: #555;
+        }
+
+        .product-item {
+            padding: 0.25rem 0;
+            border-bottom: 1px solid #eee;
+        }
+
+        .product-item:last-child {
+            border-bottom: none;
+        }
    </style>
 </head>
 <body>
@@ -127,7 +142,32 @@
        </div>
    </div>

+    <script src="/socket.io/socket.io.js"></script>
    <script>
+        const socket = io({
+            transports: ['websocket']
+        });
+
+        socket.on('connect', () => {
+            console.log('🔌 Connected to server via WebSocket');
+        });
+
+        socket.on('categoriesUpdated', () => {
+            console.log('🔄 Categories updated, reloading tree...');
+            loadCategories();
+        });
+
+        socket.on('categoryProductsUpdated', ({ id }) => {
+            console.log(`🔄 Products for category ${id} updated, reloading...`);
+            // Find the specific category element and reload its products
+            // Since we don't have easy access to instances, we could reload the whole tree
+            // or dispatch a custom event. For simplicity, we'll reload the tree for now
+            // but ideally we'd target the specific DOM element.
+            
+            // Better approach: emit event that specific components can listen to
+            document.dispatchEvent(new CustomEvent('productsUpdated', { detail: { id } }));
+        });
+
        async function loadCategories() {
            try {
                const response = await fetch('/api/categories');
@@ -194,6 +234,62 @@
            
            header.appendChild(info);
            div.appendChild(header);
+
+            // Products
+            const productsDiv = document.createElement('div');
+            productsDiv.className = 'category-products';
+            productsDiv.innerHTML = '<small>Loading products...</small>';
+            div.appendChild(productsDiv);
+
+            // Load products
+            const loadProducts = () => {
+                productsDiv.innerHTML = '<small>Loading products...</small>';
+                productsDiv.style.display = 'block';
+                
+                fetch(`/api/categories/${category.kKategorie}/products`)
+                    .then(res => res.ok ? res.json() : [])
+                    .then(products => {
+                        productsDiv.innerHTML = '';
+                        if (products.length === 0) {
+                            productsDiv.style.display = 'none';
+                            return;
+                        }
+                        
+                        const ul = document.createElement('ul');
+                        ul.style.listStyle = 'none';
+                        
+                        products.slice(0, 3).forEach(p => {
+                            const li = document.createElement('li');
+                            li.className = 'product-item';
+                            li.textContent = `📦 ${p.cName}`;
+                            ul.appendChild(li);
+                        });
+
+                        if (products.length > 3) {
+                            const more = document.createElement('li');
+                            more.className = 'product-item';
+                            more.style.fontStyle = 'italic';
+                            more.textContent = `...and ${products.length - 3} more`;
+                            ul.appendChild(more);
+                        }
+
+                        productsDiv.appendChild(ul);
+                    })
+                    .catch(() => {
+                        productsDiv.style.display = 'none';
+                    });
+            };
+
+            loadProducts();
+
+            // Listen for updates
+            const updateHandler = (e) => {
+                if (e.detail.id === category.kKategorie) {
+                    console.log(`✨ refreshing products for category ${category.kKategorie}`);
+                    loadProducts();
+                }
+            };
+            document.addEventListener('productsUpdated', updateHandler);
            
            // Children
            if (category.children && category.children.length > 0) {
--- a/index.js
+++ b/index.js
@@ -1,21 +1,26 @@
 import categorySyncer from './category-syncer.js';
 import pictureSyncer from './picture-syncer.js';
+import categoryProductsSyncer from './category-products-syncer.js';
 import { startServer } from './server.js';

 categorySyncer.on('synced', async ({ tree, unprunedTree, changed }) => {
    if (changed) {
        console.log('🎉 Event received: Category tree updated! Root nodes:', tree.length);
    } else {
-        console.log('🎉 Event received: Sync finished (no changes). Checking images...');
+        console.log('🎉 Event received: Sync finished (no changes). Checking images and products...');
    }

-    // Extract all kBild IDs from unpruned tree (includes all categories)
+    // Extract all kBild IDs and kKategorie IDs from unpruned tree
    const imageIds = [];
+    const categoryIds = [];
    const traverse = (nodes) => {
        for (const node of nodes) {
            if (node.kBild) {
                imageIds.push(node.kBild);
            }
+            if (node.kKategorie) {
+                categoryIds.push(node.kKategorie);
+            }
            if (node.children && node.children.length > 0) {
                traverse(node.children);
            }
@@ -23,8 +28,14 @@ categorySyncer.on('synced', async ({ tree, unprunedTree, changed }) => {
    };
    traverse(unprunedTree);

-    console.log(`🔍 Found ${imageIds.length} images in category tree.`);
+    console.log(`🔍 Found ${imageIds.length} images and ${categoryIds.length} categories.`);
+    
    await pictureSyncer.syncImages(imageIds, 'categories');
+    await categoryProductsSyncer.syncProducts(categoryIds);
+});
+
+categoryProductsSyncer.on('categoryUpdated', ({ id, products }) => {
+    console.log(`📝 Category ${id} updated. Products count: ${products.length}`);
 });

 // Trigger immediate sync
@@ -43,4 +54,4 @@ process.on('SIGINT', () => {
 });

 // Start Express server
-startServer(categorySyncer);
+startServer(categorySyncer, categoryProductsSyncer);
--- a/server.js
+++ b/server.js
@@ -1,4 +1,6 @@
 import express from 'express';
+import { createServer } from 'http';
+import { Server } from 'socket.io';
 import path from 'path';
 import { fileURLToPath } from 'url';
 import fs from 'fs/promises';
@@ -6,8 +8,11 @@ import fs from 'fs/promises';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);

-export function startServer(categorySyncer) {
+export function startServer(categorySyncer, categoryProductsSyncer) {
    const app = express();
+    const httpServer = createServer(app);
+    const io = new Server(httpServer);
+
    const PORT = process.env.SERVER_PORT || 3000;
    const HOST = process.env.SERVER_HOST || '0.0.0.0';
    const CACHE_DIR = process.env.CACHE_LOCATION || './cache';
@@ -15,7 +20,8 @@ export function startServer(categorySyncer) {
    // Cache for ETags and data
    const cache = {
        categories: { etag: null, data: null },
-        html: { etag: null, data: null }
+        html: { etag: null, data: null },
+        products: new Map() // id -> { etag, data }
    };

    // Function to calculate ETag for categories
@@ -34,6 +40,24 @@ export function startServer(categorySyncer) {
        }
    }

+    // Function to calculate ETag for a specific category's products
+    async function updateProductCache(id) {
+        try {
+            const productPath = path.join(CACHE_DIR, 'products', `category_${id}.json`);
+            const data = await fs.readFile(productPath, 'utf-8');
+            const crypto = await import('crypto');
+            const etag = crypto.createHash('md5').update(data).digest('hex');
+            cache.products.set(id, { etag, data });
+        } catch (err) {
+            // If file missing, remove from cache
+            if (err.code === 'ENOENT') {
+                cache.products.delete(id);
+            } else {
+                console.error(`Error updating product cache for category ${id}:`, err);
+            }
+        }
+    }
+
    // Function to calculate ETag for HTML
    async function updateHtmlCache() {
        try {
@@ -56,10 +80,27 @@ export function startServer(categorySyncer) {
        categorySyncer.on('synced', ({ changed }) => {
            if (changed) {
                updateCategoriesCache();
+                io.emit('categoriesUpdated');
            }
        });
    }

+    // Update product cache when category products update
+    if (categoryProductsSyncer) {
+        categoryProductsSyncer.on('categoryUpdated', ({ id }) => {
+            updateProductCache(id);
+            io.emit('categoryProductsUpdated', { id });
+        });
+    }
+
+    // Socket.io connection
+    io.on('connection', (socket) => {
+        console.log('🔌 Client connected');
+        socket.on('disconnect', () => {
+            console.log('🔌 Client disconnected');
+        });
+    });
+
    // Serve category tree JSON (with ETag for conditional caching)
    app.get('/api/categories', async (req, res) => {
        try {
@@ -78,6 +119,38 @@ export function startServer(categorySyncer) {
        }
    });

+    // Serve category products JSON (with ETag)
+    app.get('/api/categories/:id/products', async (req, res) => {
+        try {
+            const id = parseInt(req.params.id);
+            
+            // Lazy load if not in cache
+            if (!cache.products.has(id)) {
+                await updateProductCache(id);
+            }
+
+            const cached = cache.products.get(id);
+
+            if (!cached) {
+                return res.status(404).json({ error: 'Category products not found' });
+            }
+
+            // Check if client has cached version
+            if (req.headers['if-none-match'] === cached.etag) {
+                return res.status(304).end(); // Not Modified
+            }
+
+            // Set cache headers with ETag
+            res.set('Cache-Control', 'public, max-age=60, must-revalidate');
+            res.set('ETag', cached.etag);
+            
+            res.json(JSON.parse(cached.data));
+        } catch (err) {
+            console.error(`Error serving products for category ${req.params.id}:`, err);
+            res.status(500).json({ error: 'Failed to load products' });
+        }
+    });
+
    // Serve category images (long cache - images rarely change)
    app.get('/img/cat/:id.avif', (req, res) => {
        const { id } = req.params;
@@ -113,7 +186,7 @@ export function startServer(categorySyncer) {
        }
    });

-    app.listen(PORT, HOST, () => {
+    httpServer.listen(PORT, HOST, () => {
        console.log(`🌐 Server running on http://${HOST}:${PORT}`);
    });
 }