789 lines
31 KiB
JavaScript
789 lines
31 KiB
JavaScript
const generateRobotsTxt = (baseUrl) => {
|
||
// Ensure URLs are properly formatted
|
||
const canonicalUrl = baseUrl.endsWith('/') ? baseUrl.slice(0, -1) : baseUrl;
|
||
|
||
const robotsTxt = `User-agent: *
|
||
Allow: /
|
||
Sitemap: ${canonicalUrl}/sitemap.xml
|
||
Crawl-delay: 0
|
||
`;
|
||
|
||
return robotsTxt;
|
||
};
|
||
|
||
// Helper function to determine unit pricing data based on product data
|
||
const determineUnitPricingData = (product) => {
|
||
const result = {
|
||
unit_pricing_measure: null,
|
||
unit_pricing_base_measure: null
|
||
};
|
||
|
||
// Unit mapping from German to Google Shopping accepted units
|
||
const unitMapping = {
|
||
// Volume (German -> Google)
|
||
'Milliliter': 'ml',
|
||
'milliliter': 'ml',
|
||
'ml': 'ml',
|
||
'Liter': 'l',
|
||
'liter': 'l',
|
||
'l': 'l',
|
||
'Zentiliter': 'cl',
|
||
'zentiliter': 'cl',
|
||
'cl': 'cl',
|
||
|
||
// Weight (German -> Google)
|
||
'Gramm': 'g',
|
||
'gramm': 'g',
|
||
'g': 'g',
|
||
'Kilogramm': 'kg',
|
||
'kilogramm': 'kg',
|
||
'kg': 'kg',
|
||
'Milligramm': 'mg',
|
||
'milligramm': 'mg',
|
||
'mg': 'mg',
|
||
|
||
// Length (German -> Google)
|
||
'Meter': 'm',
|
||
'meter': 'm',
|
||
'm': 'm',
|
||
'Zentimeter': 'cm',
|
||
'zentimeter': 'cm',
|
||
'cm': 'cm',
|
||
|
||
// Count (German -> Google)
|
||
'Stück': 'ct',
|
||
'stück': 'ct',
|
||
'Stk': 'ct',
|
||
'stk': 'ct',
|
||
'ct': 'ct',
|
||
'Blatt': 'sheet',
|
||
'blatt': 'sheet',
|
||
'sheet': 'sheet'
|
||
};
|
||
|
||
// Helper function to convert German unit to Google Shopping unit
|
||
const convertUnit = (unit) => {
|
||
if (!unit) return null;
|
||
const trimmedUnit = unit.trim();
|
||
return unitMapping[trimmedUnit] || trimmedUnit.toLowerCase();
|
||
};
|
||
|
||
// unit_pricing_measure: The quantity unit of the product as it's sold
|
||
if (product.fEinheitMenge && product.cEinheit) {
|
||
const amount = parseFloat(product.fEinheitMenge);
|
||
const unit = convertUnit(product.cEinheit);
|
||
|
||
if (amount > 0 && unit) {
|
||
result.unit_pricing_measure = `${amount} ${unit}`;
|
||
}
|
||
}
|
||
|
||
// unit_pricing_base_measure: The base quantity unit for unit pricing
|
||
if (product.cGrundEinheit && product.cGrundEinheit.trim()) {
|
||
const baseUnit = convertUnit(product.cGrundEinheit);
|
||
if (baseUnit) {
|
||
// Base measure usually needs a quantity (like 100g, 1l, etc.)
|
||
// If it's just a unit, we'll add a default quantity
|
||
if (baseUnit.match(/^[a-z]+$/)) {
|
||
// For weight/volume units, use standard base quantities
|
||
if (['g', 'kg', 'mg'].includes(baseUnit)) {
|
||
result.unit_pricing_base_measure = baseUnit === 'kg' ? '1 kg' : '100 g';
|
||
} else if (['ml', 'l', 'cl'].includes(baseUnit)) {
|
||
result.unit_pricing_base_measure = baseUnit === 'l' ? '1 l' : '100 ml';
|
||
} else if (['m', 'cm'].includes(baseUnit)) {
|
||
result.unit_pricing_base_measure = baseUnit === 'm' ? '1 m' : '100 cm';
|
||
} else {
|
||
result.unit_pricing_base_measure = `1 ${baseUnit}`;
|
||
}
|
||
} else {
|
||
result.unit_pricing_base_measure = baseUnit;
|
||
}
|
||
}
|
||
}
|
||
|
||
return result;
|
||
};
|
||
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
|
||
const generateProductsXml = (allProductsData = [], baseUrl, config) => {
|
||
const currentDate = new Date().toISOString();
|
||
|
||
// Validate input
|
||
if (!Array.isArray(allProductsData) || allProductsData.length === 0) {
|
||
throw new Error("No valid product data provided");
|
||
}
|
||
|
||
// Category mapping function
|
||
const getGoogleProductCategory = (categoryId) => {
|
||
const categoryMappings = {
|
||
// Seeds & Plants
|
||
689: "543561", // Seeds (Saatgut)
|
||
706: "543561", // Stecklinge (cuttings) – ebenfalls Pflanzen/Saatgut
|
||
376: "2802", // Grow-Sets – Pflanzen- & Kräuteranbausets
|
||
|
||
// Headshop & Accessories
|
||
709: "4082", // Headshop – Rauchzubehör
|
||
711: "4082", // Headshop > Bongs – Rauchzubehör
|
||
714: "4082", // Headshop > Bongs > Zubehör – Rauchzubehör
|
||
748: "4082", // Headshop > Bongs > Köpfe – Rauchzubehör
|
||
749: "4082", // Headshop > Bongs > Chillums/Diffusoren/Kupplungen – Rauchzubehör
|
||
896: "3151", // Headshop > Vaporizer – Vaporizer
|
||
710: "5109", // Headshop > Grinder – Gewürzmühlen (Küchenhelfer)
|
||
|
||
// Measuring & Packaging
|
||
186: "5631", // Headshop > Wiegen & Verpacken – Aufbewahrung/Zubehör
|
||
187: "4767", // Headshop > Waagen – Personenwaagen (Medizinisch)
|
||
346: "7118", // Headshop > Vakuumbeutel – Vakuumierer-Beutel
|
||
355: "606", // Headshop > Boveda & Integra Boost – Luftentfeuchter (nächstmögliche)
|
||
407: "3561", // Headshop > Grove Bags – Aufbewahrungsbehälter
|
||
449: "1496", // Headshop > Cliptütchen – Lebensmittelverpackungsmaterial
|
||
539: "3110", // Headshop > Gläser & Dosen – Lebensmittelbehälter
|
||
|
||
// Lighting & Equipment
|
||
694: "3006", // Lampen – Lampen (Beleuchtung)
|
||
261: "3006", // Zubehör > Lampenzubehör – Lampen
|
||
|
||
// Plants & Growing
|
||
691: "500033", // Dünger – Dünger
|
||
692: "5633", // Zubehör > Dünger-Zubehör – Zubehör für Gartenarbeit
|
||
693: "5655", // Zelte – Zelte
|
||
|
||
// Pots & Containers
|
||
219: "113", // Töpfe – Blumentöpfe & Pflanzgefäße
|
||
220: "3173", // Töpfe > Untersetzer – Gartentopfuntersetzer und Trays
|
||
301: "113", // Töpfe > Stofftöpfe – (Blumentöpfe/Pflanzgefäße)
|
||
317: "113", // Töpfe > Air-Pot – (Blumentöpfe/Pflanzgefäße)
|
||
364: "113", // Töpfe > Kunststofftöpfe – (Blumentöpfe/Pflanzgefäße)
|
||
292: "3568", // Bewässerung > Trays & Fluttische – Bewässerungssysteme
|
||
|
||
// Ventilation & Climate
|
||
703: "2802", // Grow-Sets > Abluft-Sets – (verwendet Pflanzen-Kräuter-Anbausets)
|
||
247: "1700", // Belüftung – Ventilatoren (Klimatisierung)
|
||
214: "1700", // Belüftung > Umluft-Ventilatoren – Ventilatoren
|
||
308: "1700", // Belüftung > Ab- und Zuluft – Ventilatoren
|
||
609: "1700", // Belüftung > Ab- und Zuluft > Schalldämpfer – Ventilatoren
|
||
248: "1700", // Belüftung > Aktivkohlefilter – Ventilatoren (nächstmögliche)
|
||
392: "1700", // Belüftung > Ab- und Zuluft > Zuluftfilter – Ventilatoren
|
||
658: "606", // Belüftung > Luftbe- und -entfeuchter – Luftentfeuchter
|
||
310: "2802", // Anzucht > Heizmatten – Pflanzen- & Kräuteranbausets
|
||
379: "5631", // Belüftung > Geruchsneutralisation – Haushaltsbedarf: Aufbewahrung
|
||
|
||
// Irrigation & Watering
|
||
221: "3568", // Bewässerung – Bewässerungssysteme (Gesamt)
|
||
250: "6318", // Bewässerung > Schläuche – Gartenschläuche
|
||
297: "500100", // Bewässerung > Pumpen – Bewässerung-/Sprinklerpumpen
|
||
354: "3780", // Bewässerung > Sprüher – Sprinkler & Sprühköpfe
|
||
372: "3568", // Bewässerung > AutoPot – Bewässerungssysteme
|
||
389: "3568", // Bewässerung > Blumat – Bewässerungssysteme
|
||
405: "6318", // Bewässerung > Schläuche – Gartenschläuche
|
||
425: "3568", // Bewässerung > Wassertanks – Bewässerungssysteme
|
||
480: "3568", // Bewässerung > Tropfer – Bewässerungssysteme
|
||
519: "3568", // Bewässerung > Pumpsprüher – Bewässerungssysteme
|
||
|
||
// Growing Media & Soils
|
||
242: "543677", // Böden – Gartenerde
|
||
243: "543677", // Böden > Erde – Gartenerde
|
||
269: "543677", // Böden > Kokos – Gartenerde
|
||
580: "543677", // Böden > Perlite & Blähton – Gartenerde
|
||
|
||
// Propagation & Starting
|
||
286: "2802", // Anzucht – Pflanzen- & Kräuteranbausets
|
||
298: "2802", // Anzucht > Steinwolltrays – Pflanzen- & Kräuteranbausets
|
||
421: "2802", // Anzucht > Vermehrungszubehör – Pflanzen- & Kräuteranbausets
|
||
489: "2802", // Anzucht > EazyPlug & Jiffy – Pflanzen- & Kräuteranbausets
|
||
359: "3103", // Anzucht > Gewächshäuser – Gewächshäuser
|
||
|
||
// Tools & Equipment
|
||
373: "3568", // Bewässerung > GrowTool – Bewässerungssysteme
|
||
403: "3999", // Bewässerung > Messbecher & mehr – Messbecher & Dosierlöffel
|
||
259: "756", // Zubehör > Ernte & Verarbeitung > Pressen – Nudelmaschinen
|
||
280: "2948", // Zubehör > Ernte & Verarbeitung > Erntescheeren – Küchenmesser
|
||
258: "684", // Zubehör > Ernte & Verarbeitung – Abfallzerkleinerer
|
||
278: "5057", // Zubehör > Ernte & Verarbeitung > Extraktion – Slush-Eis-Maschinen
|
||
302: "7332", // Zubehör > Ernte & Verarbeitung > Erntemaschinen – Gartenmaschinen
|
||
|
||
// Hardware & Plumbing
|
||
222: "3568", // Bewässerung > PE-Teile – Bewässerungssysteme
|
||
374: "1700", // Belüftung > Ab- und Zuluft > Verbindungsteile – Ventilatoren
|
||
|
||
// Electronics & Control
|
||
314: "1700", // Belüftung > Steuergeräte – Ventilatoren
|
||
408: "1700", // Belüftung > Steuergeräte > GrowControl – Ventilatoren
|
||
344: "1207", // Zubehör > Messgeräte – Messwerkzeuge & Messwertgeber
|
||
555: "4555", // Zubehör > Anbauzubehör > Mikroskope – Mikroskope
|
||
|
||
// Camping & Outdoor
|
||
226: "5655", // Zubehör > Zeltzubehör – Zelte
|
||
|
||
// Plant Care & Protection
|
||
239: "4085", // Zubehör > Anbauzubehör > Pflanzenschutz – Herbizide
|
||
240: "5633", // Zubehör > Anbauzubehör – Zubehör für Gartenarbeit
|
||
|
||
// Office & Media
|
||
424: "4377", // Zubehör > Anbauzubehör > Etiketten & Schilder – Etiketten & Anhängerschilder
|
||
387: "543541", // Zubehör > Anbauzubehör > Literatur – Bücher
|
||
|
||
// General categories
|
||
686: "1700", // Belüftung > Aktivkohlefilter > Zubehör – Ventilatoren
|
||
741: "1700", // Belüftung > Ab- und Zuluft > Zubehör – Ventilatoren
|
||
294: "3568", // Bewässerung > Zubehör – Bewässerungssysteme
|
||
695: "5631", // Zubehör – Haushaltsbedarf: Aufbewahrung
|
||
293: "5631", // Zubehör > Ernte & Verarbeitung > Trockennetze – Haushaltsbedarf: Aufbewahrung
|
||
4: "5631", // Zubehör > Anbauzubehör > Sonstiges – Haushaltsbedarf: Aufbewahrung
|
||
450: "5631", // Zubehör > Anbauzubehör > Restposten – Haushaltsbedarf: Aufbewahrung
|
||
};
|
||
|
||
const categoryId_str = categoryMappings[categoryId] || "5631"; // Default to Haushaltsbedarf: Aufbewahrung
|
||
|
||
// Validate that the category ID is not empty
|
||
if (!categoryId_str || categoryId_str.trim() === "") {
|
||
return "5631"; // Haushaltsbedarf: Aufbewahrung
|
||
}
|
||
|
||
return categoryId_str;
|
||
};
|
||
|
||
let productsXml = `<?xml version="1.0" encoding="UTF-8"?>
|
||
<rss xmlns:g="http://base.google.com/ns/1.0" version="2.0">
|
||
<channel>
|
||
<title>${config.descriptions.de.short}</title>
|
||
<link>${baseUrl}</link>
|
||
<description>${config.descriptions.de.short}</description>
|
||
<lastBuildDate>${currentDate}</lastBuildDate>
|
||
<language>de-DE</language>`;
|
||
|
||
// Helper function to clean text content of problematic characters
|
||
const cleanTextContent = (text) => {
|
||
if (!text) return "";
|
||
|
||
return text.toString()
|
||
// Remove HTML tags
|
||
.replace(/<[^>]*>/g, "")
|
||
// Remove non-printable characters and control characters
|
||
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]/g, '')
|
||
// Remove BOM and other Unicode formatting characters
|
||
.replace(/[\uFEFF\u200B-\u200D\u2060]/g, '')
|
||
// Replace multiple whitespace with single space
|
||
.replace(/\s+/g, ' ')
|
||
// Remove leading/trailing whitespace
|
||
.trim();
|
||
};
|
||
|
||
// Helper function to properly escape XML content and remove invalid characters
|
||
const escapeXml = (unsafe) => {
|
||
if (!unsafe) return "";
|
||
|
||
// Convert to string and remove invalid XML characters
|
||
const cleaned = unsafe.toString()
|
||
// Remove control characters except tab (0x09), newline (0x0A), and carriage return (0x0D)
|
||
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '')
|
||
// Remove invalid Unicode characters and surrogates
|
||
.replace(/[\uD800-\uDFFF]/g, '')
|
||
// Remove other problematic characters
|
||
.replace(/[\uFFFE\uFFFF]/g, '')
|
||
// Normalize whitespace
|
||
.replace(/\s+/g, ' ')
|
||
.trim();
|
||
|
||
// Escape XML entities
|
||
return cleaned
|
||
.replace(/&/g, "&")
|
||
.replace(/</g, "<")
|
||
.replace(/>/g, ">")
|
||
.replace(/"/g, """)
|
||
.replace(/'/g, "'");
|
||
};
|
||
|
||
let processedCount = 0;
|
||
let skippedCount = 0;
|
||
|
||
// Track skip reasons with counts and product lists
|
||
const skipReasons = {
|
||
noProductOrSeoName: { count: 0, products: [] },
|
||
excludedCategory: { count: 0, products: [] },
|
||
excludedTermsTitle: { count: 0, products: [] },
|
||
excludedTermsDescription: { count: 0, products: [] },
|
||
missingGTIN: { count: 0, products: [] },
|
||
invalidGTINChecksum: { count: 0, products: [] },
|
||
missingPicture: { count: 0, products: [] },
|
||
missingWeight: { count: 0, products: [] },
|
||
insufficientDescription: { count: 0, products: [] },
|
||
nameTooShort: { count: 0, products: [] },
|
||
outOfStock: { count: 0, products: [] },
|
||
zeroPriceOrInvalid: { count: 0, products: [] },
|
||
processingError: { count: 0, products: [] }
|
||
};
|
||
|
||
// Legacy arrays for backward compatibility
|
||
const productsNeedingWeight = [];
|
||
const productsNeedingDescription = [];
|
||
|
||
// Category IDs to skip (seeds, plants, headshop items)
|
||
const skipCategoryIds = [689, 706, 709, 711, 714, 748, 749, 896, 710];
|
||
|
||
// Add each product as an item
|
||
allProductsData.forEach((product, index) => {
|
||
|
||
console.log('DEBUG '+JSON.stringify(product, null, 2));
|
||
|
||
|
||
|
||
try {
|
||
// Skip products without essential data
|
||
if (!product || !product.seoName) {
|
||
skippedCount++;
|
||
skipReasons.noProductOrSeoName.count++;
|
||
skipReasons.noProductOrSeoName.products.push({
|
||
id: product?.articleNumber || 'N/A',
|
||
name: product?.name || 'N/A',
|
||
url: product?.seoName ? `/Artikel/${product.seoName}` : 'N/A'
|
||
});
|
||
return;
|
||
}
|
||
|
||
// Skip products from excluded categories
|
||
const productCategoryId = product.categoryId || product.category_id || product.category || null;
|
||
if (productCategoryId && skipCategoryIds.includes(parseInt(productCategoryId))) {
|
||
skippedCount++;
|
||
skipReasons.excludedCategory.count++;
|
||
skipReasons.excludedCategory.products.push({
|
||
id: product.articleNumber || product.seoName,
|
||
name: product.name || 'N/A',
|
||
categoryId: productCategoryId,
|
||
url: `/Artikel/${product.seoName}`
|
||
});
|
||
return;
|
||
}
|
||
|
||
// Skip products with excluded terms in title or description
|
||
const productTitle = (product.name || "").toLowerCase();
|
||
|
||
// Get description early so we can check it for excluded terms
|
||
const productDescription = product.kurzBeschreibung || product.description || '';
|
||
|
||
const excludedTerms = {
|
||
title: ['canna', 'hash', 'marijuana', 'marihuana'],
|
||
description: ['cannabis']
|
||
};
|
||
|
||
// Check title for excluded terms
|
||
const excludedTitleTerm = excludedTerms.title.find(term => productTitle.includes(term));
|
||
if (excludedTitleTerm) {
|
||
skippedCount++;
|
||
skipReasons.excludedTermsTitle.count++;
|
||
skipReasons.excludedTermsTitle.products.push({
|
||
id: product.articleNumber || product.seoName,
|
||
name: product.name || 'N/A',
|
||
term: excludedTitleTerm,
|
||
url: `/Artikel/${product.seoName}`
|
||
});
|
||
return;
|
||
}
|
||
|
||
// Check description for excluded terms
|
||
const excludedDescTerm = excludedTerms.description.find(term => productDescription.toLowerCase().includes(term));
|
||
if (excludedDescTerm) {
|
||
skippedCount++;
|
||
skipReasons.excludedTermsDescription.count++;
|
||
skipReasons.excludedTermsDescription.products.push({
|
||
id: product.articleNumber || product.seoName,
|
||
name: product.name || 'N/A',
|
||
term: excludedDescTerm,
|
||
url: `/Artikel/${product.seoName}`
|
||
});
|
||
return;
|
||
}
|
||
|
||
// Skip products without GTIN or with invalid GTIN
|
||
if (!product.gtin || !product.gtin.toString().trim()) {
|
||
skippedCount++;
|
||
skipReasons.missingGTIN.count++;
|
||
skipReasons.missingGTIN.products.push({
|
||
id: product.articleNumber || product.seoName,
|
||
name: product.name || 'N/A',
|
||
url: `/Artikel/${product.seoName}`
|
||
});
|
||
return;
|
||
}
|
||
|
||
// Validate GTIN format and checksum
|
||
const gtinString = product.gtin.toString().trim();
|
||
|
||
// Helper function to validate GTIN with proper checksum validation
|
||
const isValidGTIN = (gtin) => {
|
||
if (!/^\d{8}$|^\d{12,14}$/.test(gtin)) return false; // Only 8, 12, 13, 14 digits allowed
|
||
|
||
const digits = gtin.split('').map(Number);
|
||
const length = digits.length;
|
||
let sum = 0;
|
||
|
||
if (length === 8) {
|
||
// EAN-8: positions 0-6, check digit at 7
|
||
// Multipliers: 3,1,3,1,3,1,3 for positions 0-6
|
||
for (let i = 0; i < 7; i++) {
|
||
const multiplier = (i % 2 === 0) ? 3 : 1;
|
||
sum += digits[i] * multiplier;
|
||
}
|
||
} else if (length === 12) {
|
||
// UPC-A: positions 0-10, check digit at 11
|
||
// Multipliers: 3,1,3,1,3,1,3,1,3,1,3 for positions 0-10
|
||
for (let i = 0; i < 11; i++) {
|
||
const multiplier = (i % 2 === 0) ? 3 : 1;
|
||
sum += digits[i] * multiplier;
|
||
}
|
||
} else if (length === 13) {
|
||
// EAN-13: positions 0-11, check digit at 12
|
||
// Multipliers: 1,3,1,3,1,3,1,3,1,3,1,3 for positions 0-11
|
||
for (let i = 0; i < 12; i++) {
|
||
const multiplier = (i % 2 === 0) ? 1 : 3;
|
||
sum += digits[i] * multiplier;
|
||
}
|
||
} else if (length === 14) {
|
||
// EAN-14: similar to EAN-13 but 14 digits
|
||
for (let i = 0; i < 13; i++) {
|
||
const multiplier = (i % 2 === 0) ? 1 : 3;
|
||
sum += digits[i] * multiplier;
|
||
}
|
||
}
|
||
const checkDigit = (10 - (sum % 10)) % 10;
|
||
return checkDigit === digits[length - 1];
|
||
};
|
||
|
||
if (!isValidGTIN(gtinString)) {
|
||
skippedCount++;
|
||
skipReasons.invalidGTINChecksum.count++;
|
||
skipReasons.invalidGTINChecksum.products.push({
|
||
id: product.articleNumber || product.seoName,
|
||
name: product.name || 'N/A',
|
||
gtin: gtinString,
|
||
url: `/Artikel/${product.seoName}`
|
||
});
|
||
return;
|
||
}
|
||
|
||
// Skip products without pictures
|
||
if (!product.pictureList || !product.pictureList.trim()) {
|
||
skippedCount++;
|
||
skipReasons.missingPicture.count++;
|
||
skipReasons.missingPicture.products.push({
|
||
id: product.articleNumber || product.seoName,
|
||
name: product.name || 'N/A',
|
||
url: `/Artikel/${product.seoName}`
|
||
});
|
||
return;
|
||
}
|
||
|
||
// Check if product has weight data - validate BEFORE building XML
|
||
if (!product.weight || isNaN(product.weight)) {
|
||
// Track products without weight
|
||
const productInfo = {
|
||
id: product.articleNumber || product.seoName,
|
||
name: product.name || 'Unnamed',
|
||
url: `/Artikel/${product.seoName}`
|
||
};
|
||
productsNeedingWeight.push(productInfo);
|
||
skipReasons.missingWeight.count++;
|
||
skipReasons.missingWeight.products.push(productInfo);
|
||
skippedCount++;
|
||
return;
|
||
}
|
||
|
||
// Check if description is missing or too short (less than 20 characters) - skip if insufficient
|
||
const originalDescription = productDescription ? cleanTextContent(productDescription) : '';
|
||
if (!originalDescription || originalDescription.length < 20) {
|
||
const productInfo = {
|
||
id: product.articleNumber || product.seoName,
|
||
name: product.name || 'Unnamed',
|
||
currentDescription: originalDescription || 'NONE',
|
||
url: `/Artikel/${product.seoName}`
|
||
};
|
||
productsNeedingDescription.push(productInfo);
|
||
skipReasons.insufficientDescription.count++;
|
||
skipReasons.insufficientDescription.products.push(productInfo);
|
||
skippedCount++;
|
||
return;
|
||
}
|
||
|
||
// Clean description for feed (remove HTML tags and limit length)
|
||
const feedDescription = cleanTextContent(productDescription).substring(0, 500);
|
||
const cleanDescription = escapeXml(feedDescription) || "Produktbeschreibung nicht verfügbar";
|
||
|
||
// Clean product name
|
||
const rawName = product.name || "Unnamed Product";
|
||
const cleanName = escapeXml(cleanTextContent(rawName)) || "Unnamed Product";
|
||
|
||
// Validate essential fields
|
||
if (!cleanName || cleanName.length < 2) {
|
||
skippedCount++;
|
||
skipReasons.nameTooShort.count++;
|
||
skipReasons.nameTooShort.products.push({
|
||
id: product.articleNumber || product.seoName,
|
||
name: rawName,
|
||
cleanedName: cleanName,
|
||
url: `/Artikel/${product.seoName}`
|
||
});
|
||
return;
|
||
}
|
||
|
||
// Generate product URL
|
||
const productUrl = `${baseUrl}/Artikel/${encodeURIComponent(product.seoName)}`;
|
||
|
||
// Generate image URL
|
||
const imageUrl = product.pictureList && product.pictureList.trim()
|
||
? `${baseUrl}/assets/images/prod${product.pictureList.split(",")[0].trim()}.jpg`
|
||
: `${baseUrl}/assets/images/nopicture.jpg`;
|
||
|
||
// Generate brand (manufacturer)
|
||
const rawBrand = product.manufacturer || config.brandName;
|
||
const brand = escapeXml(cleanTextContent(rawBrand));
|
||
|
||
// Generate condition (always new for this type of shop)
|
||
const condition = "new";
|
||
|
||
// Generate availability
|
||
const availability = product.available ? "in stock" : "out of stock";
|
||
|
||
// Skip products that are out of stock
|
||
if (!product.available) {
|
||
skippedCount++;
|
||
skipReasons.outOfStock.count++;
|
||
skipReasons.outOfStock.products.push({
|
||
id: product.articleNumber || product.seoName,
|
||
name: product.name || 'N/A',
|
||
url: `/Artikel/${product.seoName}`
|
||
});
|
||
return;
|
||
}
|
||
|
||
// Generate price (ensure it's a valid number)
|
||
const price = product.price && !isNaN(product.price)
|
||
? `${parseFloat(product.price).toFixed(2)} ${config.currency}`
|
||
: `0.00 ${config.currency}`;
|
||
|
||
// Skip products with price == 0
|
||
if (!product.price || parseFloat(product.price) === 0) {
|
||
skippedCount++;
|
||
skipReasons.zeroPriceOrInvalid.count++;
|
||
skipReasons.zeroPriceOrInvalid.products.push({
|
||
id: product.articleNumber || product.seoName,
|
||
name: product.name || 'N/A',
|
||
price: product.price,
|
||
url: `/Artikel/${product.seoName}`
|
||
});
|
||
return;
|
||
}
|
||
|
||
// Generate GTIN/EAN if available (use the already validated gtinString)
|
||
const gtin = gtinString ? escapeXml(gtinString) : null;
|
||
|
||
// Generate product ID (using articleNumber or seoName)
|
||
const rawProductId = product.articleNumber || product.seoName || `product_${Date.now()}_${Math.random().toString(36).substr(2, 5)}`;
|
||
const productId = escapeXml(rawProductId.toString().trim()) || `fallback_${Date.now()}_${Math.random().toString(36).substr(2, 5)}`;
|
||
|
||
// Get Google product category based on product's category ID
|
||
const categoryId = product.categoryId || product.category_id || product.category || null;
|
||
const googleCategory = getGoogleProductCategory(categoryId);
|
||
const escapedGoogleCategory = escapeXml(googleCategory);
|
||
|
||
// Build item XML with proper formatting (all validation passed, safe to write XML)
|
||
productsXml += `
|
||
<item>
|
||
<g:id>${productId}</g:id>
|
||
<g:title>${cleanName}</g:title>
|
||
<g:description>${cleanDescription}</g:description>
|
||
<g:link>${productUrl}</g:link>
|
||
<g:image_link>${imageUrl}</g:image_link>
|
||
<g:condition>${condition}</g:condition>
|
||
<g:availability>${availability}</g:availability>
|
||
<g:price>${price}</g:price>
|
||
<g:shipping>
|
||
<g:country>${config.country}</g:country>
|
||
<g:service>${config.shipping.defaultService}</g:service>
|
||
<g:price>${config.shipping.defaultCost}</g:price>
|
||
</g:shipping>
|
||
<g:brand>${brand}</g:brand>
|
||
<g:google_product_category>${escapedGoogleCategory}</g:google_product_category>
|
||
<g:product_type>Gartenbedarf</g:product_type>`;
|
||
|
||
// Add GTIN if available
|
||
if (gtin && gtin.trim()) {
|
||
productsXml += `
|
||
<g:gtin>${gtin}</g:gtin>`;
|
||
}
|
||
|
||
// Add weight (we know it exists at this point since we validated it earlier)
|
||
// Convert from kg to grams (multiply by 1000)
|
||
const weightInGrams = parseFloat(product.weight) * 1000;
|
||
productsXml += `
|
||
<g:shipping_weight>${weightInGrams.toFixed(2)} g</g:shipping_weight>`;
|
||
|
||
// Add unit pricing data (required by German law for many products)
|
||
const unitPricingData = determineUnitPricingData(product);
|
||
if (unitPricingData.unit_pricing_measure) {
|
||
productsXml += `
|
||
<g:unit_pricing_measure>${unitPricingData.unit_pricing_measure}</g:unit_pricing_measure>`;
|
||
}
|
||
if (unitPricingData.unit_pricing_base_measure) {
|
||
productsXml += `
|
||
<g:unit_pricing_base_measure>${unitPricingData.unit_pricing_base_measure}</g:unit_pricing_base_measure>`;
|
||
}
|
||
|
||
productsXml += `
|
||
</item>`;
|
||
|
||
processedCount++;
|
||
|
||
} catch (itemError) {
|
||
console.log(` ⚠️ Skipped product ${index + 1}: ${itemError.message}`);
|
||
skippedCount++;
|
||
skipReasons.processingError.count++;
|
||
skipReasons.processingError.products.push({
|
||
id: product?.articleNumber || product?.seoName || 'N/A',
|
||
name: product?.name || 'N/A',
|
||
error: itemError.message,
|
||
url: product?.seoName ? `/Artikel/${product.seoName}` : 'N/A'
|
||
});
|
||
}
|
||
});
|
||
|
||
productsXml += `
|
||
</channel>
|
||
</rss>`;
|
||
|
||
console.log(`\n 📊 Processing summary: ${processedCount} products included, ${skippedCount} skipped`);
|
||
|
||
// Display skip reason totals
|
||
console.log(`\n 📋 Skip Reasons Breakdown:`);
|
||
console.log(` ────────────────────────────────────────────────────────────`);
|
||
|
||
const skipReasonLabels = {
|
||
noProductOrSeoName: 'No Product or SEO Name',
|
||
excludedCategory: 'Excluded Category',
|
||
excludedTermsTitle: 'Excluded Terms in Title',
|
||
excludedTermsDescription: 'Excluded Terms in Description',
|
||
missingGTIN: 'Missing GTIN',
|
||
invalidGTINChecksum: 'Invalid GTIN Checksum',
|
||
missingPicture: 'Missing Picture',
|
||
missingWeight: 'Missing Weight',
|
||
insufficientDescription: 'Insufficient Description',
|
||
nameTooShort: 'Name Too Short',
|
||
outOfStock: 'Out of Stock',
|
||
zeroPriceOrInvalid: 'Zero or Invalid Price',
|
||
processingError: 'Processing Error'
|
||
};
|
||
|
||
let hasAnySkips = false;
|
||
Object.entries(skipReasons).forEach(([key, data]) => {
|
||
if (data.count > 0) {
|
||
hasAnySkips = true;
|
||
const label = skipReasonLabels[key] || key;
|
||
console.log(` • ${label}: ${data.count}`);
|
||
}
|
||
});
|
||
|
||
if (!hasAnySkips) {
|
||
console.log(` ✅ No products were skipped`);
|
||
}
|
||
|
||
console.log(` ────────────────────────────────────────────────────────────`);
|
||
console.log(` Total: ${skippedCount} products skipped\n`);
|
||
|
||
// Write log files for products needing attention
|
||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||
const logsDir = path.join(process.cwd(), 'logs');
|
||
|
||
// Ensure logs directory exists
|
||
if (!fs.existsSync(logsDir)) {
|
||
fs.mkdirSync(logsDir, { recursive: true });
|
||
}
|
||
|
||
// Write comprehensive skip reasons log
|
||
const skipLogPath = path.join(logsDir, `skip-reasons-${timestamp}.log`);
|
||
let skipLogContent = `# Product Skip Reasons Report
|
||
# Generated: ${new Date().toISOString()}
|
||
# Total products processed: ${processedCount}
|
||
# Total products skipped: ${skippedCount}
|
||
# Base URL: ${baseUrl}
|
||
|
||
`;
|
||
|
||
Object.entries(skipReasons).forEach(([key, data]) => {
|
||
if (data.count > 0) {
|
||
const label = skipReasonLabels[key] || key;
|
||
skipLogContent += `\n## ${label} (${data.count} products)\n`;
|
||
skipLogContent += `${'='.repeat(80)}\n`;
|
||
|
||
data.products.forEach(product => {
|
||
skipLogContent += `ID: ${product.id}\n`;
|
||
skipLogContent += `Name: ${product.name}\n`;
|
||
if (product.categoryId !== undefined) {
|
||
skipLogContent += `Category ID: ${product.categoryId}\n`;
|
||
}
|
||
if (product.term !== undefined) {
|
||
skipLogContent += `Excluded Term: ${product.term}\n`;
|
||
}
|
||
if (product.gtin !== undefined) {
|
||
skipLogContent += `GTIN: ${product.gtin}\n`;
|
||
}
|
||
if (product.currentDescription !== undefined) {
|
||
skipLogContent += `Current Description: "${product.currentDescription}"\n`;
|
||
}
|
||
if (product.cleanedName !== undefined) {
|
||
skipLogContent += `Cleaned Name: "${product.cleanedName}"\n`;
|
||
}
|
||
if (product.price !== undefined) {
|
||
skipLogContent += `Price: ${product.price}\n`;
|
||
}
|
||
if (product.error !== undefined) {
|
||
skipLogContent += `Error: ${product.error}\n`;
|
||
}
|
||
skipLogContent += `URL: ${baseUrl}${product.url}\n`;
|
||
skipLogContent += `${'-'.repeat(80)}\n`;
|
||
});
|
||
}
|
||
});
|
||
|
||
fs.writeFileSync(skipLogPath, skipLogContent, 'utf8');
|
||
console.log(` 📄 Detailed skip reasons report saved to: ${skipLogPath}`);
|
||
|
||
// Write missing weight log (for backward compatibility)
|
||
if (productsNeedingWeight.length > 0) {
|
||
const weightLogContent = `# Products Missing Weight Data
|
||
# Generated: ${new Date().toISOString()}
|
||
# Total products missing weight: ${productsNeedingWeight.length}
|
||
|
||
${productsNeedingWeight.map(product => `${product.id}\t${product.name}\t${baseUrl}${product.url}`).join('\n')}
|
||
`;
|
||
|
||
const weightLogPath = path.join(logsDir, `missing-weight-${timestamp}.log`);
|
||
fs.writeFileSync(weightLogPath, weightLogContent, 'utf8');
|
||
console.log(` ⚠️ Products missing weight (${productsNeedingWeight.length}) - saved to: ${weightLogPath}`);
|
||
}
|
||
|
||
// Write missing description log (for backward compatibility)
|
||
if (productsNeedingDescription.length > 0) {
|
||
const descLogContent = `# Products With Insufficient Description Data
|
||
# Generated: ${new Date().toISOString()}
|
||
# Total products needing description: ${productsNeedingDescription.length}
|
||
|
||
${productsNeedingDescription.map(product => `${product.id}\t${product.name}\t"${product.currentDescription}"\t${baseUrl}${product.url}`).join('\n')}
|
||
`;
|
||
|
||
const descLogPath = path.join(logsDir, `missing-description-${timestamp}.log`);
|
||
fs.writeFileSync(descLogPath, descLogContent, 'utf8');
|
||
console.log(` ⚠️ Products with insufficient description (${productsNeedingDescription.length}) - saved to: ${descLogPath}`);
|
||
}
|
||
|
||
if (productsNeedingWeight.length === 0 && productsNeedingDescription.length === 0) {
|
||
console.log(` ✅ All products have adequate weight and description data`);
|
||
}
|
||
|
||
return productsXml;
|
||
};
|
||
|
||
module.exports = {
|
||
generateRobotsTxt,
|
||
generateProductsXml,
|
||
};
|