From 3f2cad2426e8bf1d57e8a743e057d92b71bdd1c5 Mon Sep 17 00:00:00 2001 From: sebseb7 Date: Fri, 1 Aug 2025 14:53:14 +0200 Subject: [PATCH] i --- data.csv | 0 src/routes/data.js | 249 ++++++++++++++++++++++++++++----------------- 2 files changed, 154 insertions(+), 95 deletions(-) delete mode 100644 data.csv diff --git a/data.csv b/data.csv deleted file mode 100644 index e69de29..0000000 diff --git a/src/routes/data.js b/src/routes/data.js index 95d71df..e992a04 100644 --- a/src/routes/data.js +++ b/src/routes/data.js @@ -5,81 +5,27 @@ const { authenticateToken } = require('../middleware/auth'); const router = express.Router(); -// Parse CSV data -const parseCSV = () => { - try { - const csvPath = path.join(__dirname, '../../data.csv'); - const csvData = fs.readFileSync(csvPath, 'utf8'); - const lines = csvData.split('\n'); - const headers = lines[0].split(';').map(h => h.replace(/"/g, '')); - - const transactions = []; - for (let i = 1; i < lines.length; i++) { - const line = lines[i]; - if (!line.trim()) continue; - - // Parse CSV line (handle semicolon-separated values with quotes) - const values = []; - let current = ''; - let inQuotes = false; - - for (let j = 0; j < line.length; j++) { - const char = line[j]; - if (char === '"') { - inQuotes = !inQuotes; - } else if (char === ';' && !inQuotes) { - values.push(current); - current = ''; - } else { - current += char; - } - } - values.push(current); // Add last value - - if (values.length >= headers.length) { - const transaction = {}; - headers.forEach((header, index) => { - transaction[header] = values[index] || ''; - }); - - // Parse date and amount - if (transaction['Buchungstag']) { - const dateParts = transaction['Buchungstag'].split('.'); - if (dateParts.length === 3) { - // Convert DD.MM.YY to proper date - const day = dateParts[0]; - const month = dateParts[1]; - const year = '20' + dateParts[2]; // Assuming 20xx - transaction.parsedDate = new Date(year, month - 1, day); - transaction.monthYear = `${year}-${month.padStart(2, '0')}`; - } - } - - // Parse amount - if (transaction['Betrag']) { - const amount = transaction['Betrag'].replace(',', '.').replace(/[^-0-9.]/g, ''); - transaction.numericAmount = parseFloat(amount) || 0; - } - - transactions.push(transaction); - } - } - - return transactions; - } catch (error) { - console.error('Error parsing CSV:', error); - return []; - } -}; +// Old CSV parsing removed - now using database-based CSV import -// Get available months -router.get('/months', authenticateToken, (req, res) => { +// Get available months from database +router.get('/months', authenticateToken, async (req, res) => { try { - const transactions = parseCSV(); - const months = [...new Set(transactions - .filter(t => t.monthYear) - .map(t => t.monthYear) - )].sort().reverse(); // Newest first + const { executeQuery } = require('../config/database'); + + const query = ` + SELECT DISTINCT + FORMAT(combined.date_col, 'yyyy-MM') as month_year + FROM ( + SELECT buchungsdatum as date_col FROM fibdash.AccountingItems WHERE buchungsdatum IS NOT NULL + UNION ALL + SELECT parsed_date as date_col FROM fibdash.CSVTransactions WHERE parsed_date IS NOT NULL + ) combined + WHERE combined.date_col IS NOT NULL + ORDER BY month_year DESC + `; + + const result = await executeQuery(query); + const months = result.recordset.map(row => row.month_year); res.json({ months }); } catch (error) { @@ -155,7 +101,95 @@ const getJTLTransactions = async () => { router.get('/transactions/:timeRange', authenticateToken, async (req, res) => { try { const { timeRange } = req.params; - const transactions = parseCSV(); + + // Get CSV transactions with JTL matches AND JTL-only transactions + const { executeQuery } = require('../config/database'); + const query = ` + SELECT + csv.buchungstag as 'Buchungstag', + csv.wertstellung as 'Valutadatum', + csv.umsatzart as 'Buchungstext', + csv.verwendungszweck as 'Verwendungszweck', + csv.beguenstigter_zahlungspflichtiger as 'Beguenstigter/Zahlungspflichtiger', + csv.kontonummer_iban as 'Kontonummer/IBAN', + csv.bic as 'BIC (SWIFT-Code)', + csv.betrag_original as 'Betrag', + csv.waehrung as 'Waehrung', + csv.numeric_amount as numericAmount, + csv.parsed_date, + FORMAT(csv.parsed_date, 'yyyy-MM') as monthYear, + jtl.kZahlungsabgleichUmsatz as jtlId, + CASE WHEN jtl.kZahlungsabgleichUmsatz IS NOT NULL THEN 1 ELSE 0 END as hasJTL, + k.name as kreditor_name, + k.kreditorId as kreditor_id, + k.is_banking as kreditor_is_banking, + bat.assigned_kreditor_id, + ak.name as assigned_kreditor_name, + 0 as isJTLOnly, + 1 as isFromCSV + FROM fibdash.CSVTransactions csv + LEFT JOIN eazybusiness.dbo.tZahlungsabgleichUmsatz jtl ON ( + ABS(csv.numeric_amount - jtl.fBetrag) < 0.01 AND + ABS(DATEDIFF(day, csv.parsed_date, jtl.dBuchungsdatum)) <= 1 + ) + LEFT JOIN fibdash.Kreditor k ON csv.kontonummer_iban = k.iban + LEFT JOIN fibdash.BankingAccountTransactions bat ON csv.id = bat.csv_transaction_id + LEFT JOIN fibdash.Kreditor ak ON bat.assigned_kreditor_id = ak.id + + UNION ALL + + SELECT + FORMAT(jtl.dBuchungsdatum, 'dd.MM.yy') as 'Buchungstag', + FORMAT(jtl.dBuchungsdatum, 'dd.MM.yy') as 'Valutadatum', + 'JTL Transaction' as 'Buchungstext', + jtl.cVerwendungszweck as 'Verwendungszweck', + jtl.cName as 'Beguenstigter/Zahlungspflichtiger', + '' as 'Kontonummer/IBAN', + '' as 'BIC (SWIFT-Code)', + FORMAT(jtl.fBetrag, 'N2', 'de-DE') as 'Betrag', + '' as 'Waehrung', + jtl.fBetrag as numericAmount, + jtl.dBuchungsdatum as parsed_date, + FORMAT(jtl.dBuchungsdatum, 'yyyy-MM') as monthYear, + jtl.kZahlungsabgleichUmsatz as jtlId, + 1 as hasJTL, + NULL as kreditor_name, + NULL as kreditor_id, + NULL as kreditor_is_banking, + NULL as assigned_kreditor_id, + NULL as assigned_kreditor_name, + 1 as isJTLOnly, + 0 as isFromCSV + FROM eazybusiness.dbo.tZahlungsabgleichUmsatz jtl + WHERE NOT EXISTS ( + SELECT 1 FROM fibdash.CSVTransactions csv + WHERE ABS(csv.numeric_amount - jtl.fBetrag) < 0.01 + AND ABS(DATEDIFF(day, csv.parsed_date, jtl.dBuchungsdatum)) <= 1 + ) + + ORDER BY parsed_date DESC + `; + + const result = await executeQuery(query); + const transactions = result.recordset.map(transaction => ({ + ...transaction, + parsedDate: new Date(transaction.parsed_date), + hasJTL: Boolean(transaction.hasJTL), + isFromCSV: true, + jtlDatabaseAvailable: true, + hasKreditor: !!transaction.kreditor_name, + kreditor: transaction.kreditor_name ? { + name: transaction.kreditor_name, + kreditorId: transaction.kreditor_id, + is_banking: Boolean(transaction.kreditor_is_banking) + } : null, + assigned_kreditor: transaction.assigned_kreditor_name ? { + name: transaction.assigned_kreditor_name, + id: transaction.assigned_kreditor_id + } : null, + pdfs: [], + links: [] + })); let filteredTransactions = []; let periodDescription = ''; @@ -201,8 +235,9 @@ router.get('/transactions/:timeRange', authenticateToken, async (req, res) => { try { jtlTransactions = await getJTLTransactions(); jtlDatabaseAvailable = true; + console.log('DEBUG: JTL database connected, found', jtlTransactions.length, 'transactions'); } catch (error) { - console.log('JTL database not available, continuing without JTL data'); + console.log('JTL database not available, continuing without JTL data:', error.message); jtlDatabaseAvailable = false; } @@ -247,11 +282,26 @@ router.get('/transactions/:timeRange', authenticateToken, async (req, res) => { } // Add JTL status and Kreditor information to each CSV transaction - const transactionsWithJTL = monthTransactions.map(transaction => { + const transactionsWithJTL = monthTransactions.map((transaction, index) => { // Try to match by amount and date (approximate matching) const amount = transaction.numericAmount; const transactionDate = transaction.parsedDate; + // Debug first transaction + if (index === 0) { + console.log('DEBUG First CSV transaction:', { + amount: amount, + transactionDate: transactionDate, + jtlMonthTransactionsCount: jtlMonthTransactions.length + }); + if (jtlMonthTransactions.length > 0) { + console.log('DEBUG First JTL transaction:', { + amount: parseFloat(jtlMonthTransactions[0].fBetrag), + date: new Date(jtlMonthTransactions[0].dBuchungsdatum) + }); + } + } + const jtlMatch = jtlMonthTransactions.find(jtl => { const jtlAmount = parseFloat(jtl.fBetrag) || 0; const jtlDate = new Date(jtl.dBuchungsdatum); @@ -263,6 +313,19 @@ router.get('/transactions/:timeRange', authenticateToken, async (req, res) => { transactionDate.getMonth() === jtlDate.getMonth() && transactionDate.getDate() === jtlDate.getDate(); + // Debug potential matches for first transaction + if (index === 0 && (amountMatch || dateMatch)) { + console.log('DEBUG Potential match for first transaction:', { + csvAmount: amount, + jtlAmount: jtlAmount, + amountMatch: amountMatch, + csvDate: transactionDate, + jtlDate: jtlDate, + dateMatch: dateMatch, + bothMatch: amountMatch && dateMatch + }); + } + return amountMatch && dateMatch; }); @@ -340,32 +403,27 @@ router.get('/transactions/:timeRange', authenticateToken, async (req, res) => { hasKreditor: false })); - // Combine CSV and JTL-only transactions - const allTransactions = [...transactionsWithJTL, ...unmatchedJTLTransactions] - .sort((a, b) => b.parsedDate - a.parsedDate); - // Calculate summary const summary = { - totalTransactions: allTransactions.length, - totalIncome: allTransactions + totalTransactions: filteredTransactions.length, + totalIncome: filteredTransactions .filter(t => t.numericAmount > 0) .reduce((sum, t) => sum + t.numericAmount, 0), - totalExpenses: allTransactions + totalExpenses: filteredTransactions .filter(t => t.numericAmount < 0) .reduce((sum, t) => sum + Math.abs(t.numericAmount), 0), - netAmount: allTransactions.reduce((sum, t) => sum + t.numericAmount, 0), - jtlMatches: jtlDatabaseAvailable ? allTransactions.filter(t => t.hasJTL === true && t.isFromCSV).length : undefined, - jtlMissing: jtlDatabaseAvailable ? allTransactions.filter(t => t.hasJTL === false && t.isFromCSV).length : undefined, - jtlOnly: jtlDatabaseAvailable ? allTransactions.filter(t => t.isJTLOnly).length : undefined, - csvOnly: jtlDatabaseAvailable ? allTransactions.filter(t => t.hasJTL === false && t.isFromCSV).length : undefined, - jtlDatabaseAvailable + netAmount: filteredTransactions.reduce((sum, t) => sum + t.numericAmount, 0), + timeRange: timeRange, + jtlDatabaseAvailable: true, + jtlMatches: filteredTransactions.filter(t => t.hasJTL === true && t.isFromCSV).length, + jtlMissing: filteredTransactions.filter(t => t.hasJTL === false && t.isFromCSV).length, + jtlOnly: filteredTransactions.filter(t => t.isJTLOnly === true).length, + csvOnly: filteredTransactions.filter(t => t.hasJTL === false && t.isFromCSV).length }; res.json({ - transactions: allTransactions, - summary, - timeRange, - periodDescription + transactions: filteredTransactions, + summary }); } catch (error) { console.error('Error getting transactions:', error); @@ -430,8 +488,9 @@ router.get('/datev/:timeRange', authenticateToken, async (req, res) => { try { const { timeRange } = req.params; - // Get transactions for the time period - const transactions = parseCSV(); + // TODO: Update to use database queries instead of CSV file + res.status(501).json({ error: 'DATEV export temporarily disabled - use database-based queries' }); + return; let filteredTransactions = []; let periodStart, periodEnd, filename;