From b4d202bb23378c9f15f4d295f79d332a2c830c78 Mon Sep 17 00:00:00 2001 From: sebseb7 Date: Mon, 24 Nov 2025 14:50:40 +0100 Subject: [PATCH] feat: Implement product detail syncing with rowversion-based change detection and integrate it into category product synchronization. --- index.html | 40 ++++++- src/server/utils/search-helper.js | 4 + src/services/product-detail-data-fetcher.js | 115 +++++++++++++++++++ src/syncers/category-products-syncer.js | 8 ++ src/syncers/product-detail-syncer.js | 120 ++++++++++++++++++++ 5 files changed, 281 insertions(+), 6 deletions(-) create mode 100644 src/services/product-detail-data-fetcher.js create mode 100644 src/syncers/product-detail-syncer.js diff --git a/index.html b/index.html index 3494a89..c40b57f 100644 --- a/index.html +++ b/index.html @@ -480,7 +480,15 @@ } const debouncedSearch = debounce((value) => { - socket.emit('search', value); + if (value.trim().length >= 3) { + socket.emit('search', value); + } else { + // Clear matches and collapse all categories if less than 3 chars + resetMatches(state.categories); + resetExpansion(state.categories); + collapseAllProducts(state.categories); + render(); + } }, 300); // Event Listeners @@ -495,7 +503,7 @@ clearBtn.classList.remove('visible'); } - if (value.trim()) { + if (value.trim().length >= 3) { debouncedSearch(value); } else { // Clear matches and collapse all categories @@ -607,6 +615,12 @@ }); } + function collapseAllProducts(nodes) { + nodes.forEach(node => { + node.isExpanded = false; + if (node.children) collapseAllProducts(node.children); + }); + } // Initial Load @@ -674,7 +688,7 @@ // Filtering Logic function filterTree(nodes, query) { - if (!query.trim()) return nodes; // Return original structure if no filter + if (!query.trim() || query.trim().length < 3) return nodes; // Return original structure if no filter or short query return nodes.map(node => { // Only keep if marked as having a match @@ -690,6 +704,10 @@ const name = p.cName.toLowerCase(); return words.every(w => name.includes(w)); }); + // Limit product results + if (matchingProducts.length > 21) { + matchingProducts = matchingProducts.slice(0, 21); + } } return { @@ -845,9 +863,10 @@ const ul = document.createElement('ul'); ul.style.listStyle = 'none'; - const limit = realNode.isExpanded ? category.products.length : 3; + const limit = realNode.isExpanded ? 20 : 3; + const displayProducts = category.products.slice(0, limit); - category.products.slice(0, limit).forEach(p => { + displayProducts.forEach(p => { const li = document.createElement('li'); li.className = 'product-item'; @@ -867,7 +886,16 @@ ul.appendChild(li); }); - if (!realNode.isExpanded && category.products.length > 3) { + // Show "more" if expanded and there are more than 20, OR if collapsed and there are more than 3 + if (realNode.isExpanded && category.products.length > 20) { + const more = document.createElement('li'); + more.className = 'product-item more'; + more.style.fontStyle = 'italic'; + more.textContent = `(more)`; + // Prevent click from collapsing + more.onclick = (e) => e.stopPropagation(); + ul.appendChild(more); + } else if (!realNode.isExpanded && category.products.length > 3) { const more = document.createElement('li'); more.className = 'product-item more'; more.style.fontStyle = 'italic'; diff --git a/src/server/utils/search-helper.js b/src/server/utils/search-helper.js index d292055..3828d4c 100644 --- a/src/server/utils/search-helper.js +++ b/src/server/utils/search-helper.js @@ -67,6 +67,10 @@ export async function findMatches(query, cacheDir) { matchingCategoryIds.add(node.kKategorie); } })); + + if (matchingCategoryIds.size >= 20) { + break; + } } return Array.from(matchingCategoryIds); diff --git a/src/services/product-detail-data-fetcher.js b/src/services/product-detail-data-fetcher.js new file mode 100644 index 0000000..d57fd0b --- /dev/null +++ b/src/services/product-detail-data-fetcher.js @@ -0,0 +1,115 @@ +import { createConnection } from '../utils/database.js'; +import { processInChunks, createInClause } from '../utils/database-utils.js'; + +/** + * ProductDetailDataFetcher - Handles fetching product descriptions + */ +export class ProductDetailDataFetcher { + /** + * Fetch product descriptions for given article IDs + * @param {Array} articleIds - Article IDs to fetch details for + * @param {Function} detailCallback - Callback for each detail (receives {kArtikel, cBeschreibung}) + * @param {number} chunkSize - Size of each chunk (default: 50) + * @returns {Promise} + */ + async fetchDetailsInChunks(articleIds, detailCallback, chunkSize = 50) { + let pool; + try { + pool = await createConnection(); + + await processInChunks(articleIds, chunkSize, async (chunk) => { + const list = createInClause(chunk); + + const result = await pool.request().query(` + SELECT kArtikel, cBeschreibung, bRowversion + FROM tArtikelBeschreibung + WHERE kArtikel IN (${list}) + AND kSprache = ${process.env.JTL_SPRACHE_ID} + AND kPlattform = ${process.env.JTL_PLATTFORM_ID} + AND kShop = ${process.env.JTL_SHOP_ID} + `); + + const foundIds = new Set(); + for (const record of result.recordset) { + foundIds.add(record.kArtikel); + // Convert Buffer or binary string to hex string if needed + if (Buffer.isBuffer(record.bRowversion)) { + record.bRowversion = '0x' + record.bRowversion.toString('hex').toUpperCase(); + } else if (typeof record.bRowversion === 'string' && !record.bRowversion.startsWith('0x')) { + // Assume binary string + record.bRowversion = '0x' + Buffer.from(record.bRowversion, 'binary').toString('hex').toUpperCase(); + } + + if (!record.cBeschreibung) { + console.log(`⚠️ Item ${record.kArtikel} has no description, writing empty file.`); + } + await detailCallback(record); + } + + // Check for missing items in this chunk + chunk.forEach(id => { + if (!foundIds.has(id)) { + // console.log(`⚠️ Item ${id} not found in tArtikelBeschreibung (or filtered out).`); + } + }); + }, { showProgress: true, itemName: 'details' }); + + } finally { + if (pool) await pool.close(); + } + } + + /** + * Fetch IDs of articles that have changed since a given version + * @param {Array} articleIds - Candidate article IDs + * @param {string} minRowversion - Minimum rowversion (hex string) + * @returns {Promise>} - Set of changed article IDs + */ + async fetchChangedArticleIds(articleIds, minRowversion) { + //console.log(`🔍 Checking changes for ${articleIds ? articleIds.length : 0} articles against version ${minRowversion}`); + if (!articleIds || articleIds.length === 0) return new Set(); + + // If no minRowversion, all are considered changed + if (!minRowversion) { + console.log('⚠️ No minRowversion provided, fetching all.'); + return new Set(articleIds); + } + + let pool; + const changedIds = new Set(); + + try { + pool = await createConnection(); + + await processInChunks(articleIds, 2000, async (chunk) => { + const list = createInClause(chunk); + // Convert hex string back to buffer for comparison if needed, + // but MSSQL driver usually handles 0x strings as binary. + // Let's assume minRowversion is passed as '0x...' string. + + const query = ` + SELECT kArtikel, bRowversion + FROM tArtikelBeschreibung + WHERE kArtikel IN (${list}) + AND kSprache = ${process.env.JTL_SPRACHE_ID} + AND kPlattform = ${process.env.JTL_PLATTFORM_ID} + AND kShop = ${process.env.JTL_SHOP_ID} + AND bRowversion > ${minRowversion} + `; + // console.log('Executing query:', query); + + const result = await pool.request().query(query); + + result.recordset.forEach(r => { + // console.log(`Changed item: ${r.kArtikel}, version: 0x${r.bRowversion.toString('hex').toUpperCase()}`); + changedIds.add(r.kArtikel); + }); + }, { showProgress: false }); + + if (changedIds.size > 0) console.log(`🔍 Found ${changedIds.size} changed articles.`); + return changedIds; + } finally { + if (pool) await pool.close(); + } + } +} diff --git a/src/syncers/category-products-syncer.js b/src/syncers/category-products-syncer.js index 64f1152..3b7567e 100644 --- a/src/syncers/category-products-syncer.js +++ b/src/syncers/category-products-syncer.js @@ -5,6 +5,7 @@ import { SyncQueueManager } from '../utils/sync-queue-manager.js'; import { ProductDataFetcher } from '../services/product-data-fetcher.js'; import { getExistingIds, deleteObsoleteFiles, writeJsonIfChanged, ensureDir } from '../utils/file-sync-utils.js'; import pictureSyncer from './picture-syncer.js'; +import productDetailSyncer from './product-detail-syncer.js'; class CategoryProductsSyncer extends EventEmitter { constructor() { @@ -62,6 +63,7 @@ class CategoryProductsSyncer extends EventEmitter { async _fetchAndWriteProducts(ids, dir) { const globalImageIds = new Set(); + const globalArticleIds = new Set(); await this.dataFetcher.fetchProductsInChunks(ids, async (chunkData) => { const { categoryIds, products, productImages } = chunkData; @@ -78,6 +80,7 @@ class CategoryProductsSyncer extends EventEmitter { if (productsByCategory[record.kKategorie]) { const images = productImages.get(record.kArtikel) || []; images.forEach(imgId => globalImageIds.add(imgId)); + globalArticleIds.add(record.kArtikel); productsByCategory[record.kKategorie].push({ kArtikel: record.kArtikel, @@ -105,6 +108,11 @@ class CategoryProductsSyncer extends EventEmitter { //console.log(`🖼️ Syncing ${globalImageIds.size} product images...`); await pictureSyncer.syncImages(Array.from(globalImageIds), 'products'); } + + // Sync product details for all articles found + if (globalArticleIds.size > 0) { + await productDetailSyncer.syncDetails(Array.from(globalArticleIds)); + } } } diff --git a/src/syncers/product-detail-syncer.js b/src/syncers/product-detail-syncer.js new file mode 100644 index 0000000..6043950 --- /dev/null +++ b/src/syncers/product-detail-syncer.js @@ -0,0 +1,120 @@ +import fs from 'fs/promises'; +import path from 'path'; +import { SyncQueueManager } from '../utils/sync-queue-manager.js'; +import { ProductDetailDataFetcher } from '../services/product-detail-data-fetcher.js'; +import { getExistingIds, deleteObsoleteFiles, ensureDir, writeJsonIfChanged } from '../utils/file-sync-utils.js'; + +class ProductDetailSyncer { + constructor() { + if (ProductDetailSyncer.instance) { + return ProductDetailSyncer.instance; + } + + this.syncQueue = new SyncQueueManager(); + this.dataFetcher = new ProductDetailDataFetcher(); + this.cacheBaseDir = process.env.CACHE_LOCATION || '.'; + + ProductDetailSyncer.instance = this; + } + + async syncDetails(articleIds) { + await this.syncQueue.executeSync('product-details', async () => { + await this._performSync(articleIds); + }, articleIds); + } + + async _performSync(articleIds) { + const detailsDir = path.join(this.cacheBaseDir, 'details'); + const stateFile = path.join(this.cacheBaseDir, 'product-details-state.json'); + + // Ensure directory exists + await ensureDir(detailsDir); + + // Load state + let lastSyncRowversion = null; + try { + const state = JSON.parse(await fs.readFile(stateFile, 'utf-8')); + lastSyncRowversion = state.lastSyncRowversion; + } catch (err) { + // State file might not exist yet + } + + // Get existing files + const existingIds = await getExistingIds(detailsDir, { + suffix: '.json' + }); + + const validIds = new Set(articleIds.filter(id => id !== null && id !== undefined)); + + // Delete obsolete files + await deleteObsoleteFiles( + detailsDir, + existingIds, + validIds, + (id) => `${id}.json` + ); + + // Split into missing and present + const missingIds = []; + const presentIds = []; + + for (const id of validIds) { + if (existingIds.includes(id)) { + presentIds.push(id); + } else { + missingIds.push(id); + } + } + + // Determine what to fetch + const toFetch = new Set(missingIds); + + if (presentIds.length > 0) { + // Check which present files need update based on rowversion + //console.log(`Checking changes for ${presentIds.length} present items with lastSyncRowversion: ${lastSyncRowversion}`); + const changedIds = await this.dataFetcher.fetchChangedArticleIds(presentIds, lastSyncRowversion); + //console.log(`Got ${changedIds.size} changed items from fetcher`); + changedIds.forEach(id => toFetch.add(id)); + } + + if (toFetch.size > 0) { + console.log(`📝 Syncing ${toFetch.size} product details (Missing: ${missingIds.length}, Changed: ${toFetch.size - missingIds.length})...`); + await this._fetchAndWriteDetails([...toFetch], detailsDir, stateFile, lastSyncRowversion); + } else { + //console.log(`✅ No product details to sync.`); + } + } + + async _fetchAndWriteDetails(ids, dir, stateFile, currentMaxRowversion) { + let maxRowversion = currentMaxRowversion; + + await this.dataFetcher.fetchDetailsInChunks(ids, async (record) => { + const filePath = path.join(dir, `${record.kArtikel}.json`); + + // Update max rowversion + if (record.bRowversion) { + // Simple string comparison for hex strings works for sorting/max if length is same. + // MSSQL rowversions are fixed length (8 bytes), so hex string length should be constant. + if (!maxRowversion || record.bRowversion > maxRowversion) { + maxRowversion = record.bRowversion; + } + } + + // Use writeJsonIfChanged which handles reading and comparing + // It will compare the new object with the existing JSON content + await writeJsonIfChanged(filePath, { + kArtikel: record.kArtikel, + cBeschreibung: record.cBeschreibung || null, // Ensure null is written if missing + bRowversion: record.bRowversion || null + }); + }); + + // Save new state + if (maxRowversion && maxRowversion !== currentMaxRowversion) { + await fs.writeFile(stateFile, JSON.stringify({ lastSyncRowversion: maxRowversion }, null, 2)); + } + } +} + +const instance = new ProductDetailSyncer(); +export default instance;