From bb610e04803d920df97534fcd1f8938052b48da7 Mon Sep 17 00:00:00 2001
From: sebseb7 <sebgreenbus@gmail.com>
Date: Fri, 22 Aug 2025 23:46:20 +0200
Subject: [PATCH] Add duplicate transaction check in CSV import process

- Implemented a check for existing transactions in the database to prevent duplicates during CSV imports.
- Added SQL query to count existing transactions based on key fields before insertion.
- Enhanced error handling to log and skip duplicate transactions, improving data integrity during the import process.
---
 src/routes/data/csvImport.js | 62 ++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/src/routes/data/csvImport.js b/src/routes/data/csvImport.js
index dff58c6..1cf6cf1 100644
--- a/src/routes/data/csvImport.js
+++ b/src/routes/data/csvImport.js
@@ -71,6 +71,37 @@ router.post('/test-csv-import', async (req, res) => {
           numericAmount = parseFloat(normalizedAmount) || 0;
         }
         
+        // Check for existing transaction to prevent duplicates
+        const duplicateCheckQuery = `
+          SELECT COUNT(*) as count FROM fibdash.CSVTransactions 
+          WHERE buchungstag = @buchungstag 
+            AND wertstellung = @wertstellung 
+            AND umsatzart = @umsatzart 
+            AND betrag = @betrag 
+            AND beguenstigter_zahlungspflichtiger = @beguenstigter_zahlungspflichtiger 
+            AND verwendungszweck = @verwendungszweck
+        `;
+        
+        const duplicateCheckResult = await executeQuery(duplicateCheckQuery, {
+          buchungstag: transaction['Buchungstag'] || null,
+          wertstellung: transaction['Valutadatum'] || null,
+          umsatzart: transaction['Buchungstext'] || null,
+          betrag: numericAmount,
+          beguenstigter_zahlungspflichtiger: transaction['Beguenstigter/Zahlungspflichtiger'] || null,
+          verwendungszweck: transaction['Verwendungszweck'] || null
+        });
+        
+        if (duplicateCheckResult.recordset[0].count > 0) {
+          console.log(`Skipping duplicate transaction at row ${i + 1}: ${transaction['Buchungstag']} - ${numericAmount}`);
+          errors.push({
+            row: i + 1,
+            error: 'Duplicate transaction (already exists in database)',
+            transaction: transaction
+          });
+          errorCount++;
+          continue;
+        }
+        
         const insertQuery = `
           INSERT INTO fibdash.CSVTransactions
           (buchungstag, wertstellung, umsatzart, betrag, betrag_original, waehrung,
@@ -246,6 +277,37 @@ router.post('/import-csv-transactions', authenticateToken, async (req, res) => {
           numericAmount = parseFloat(normalizedAmount) || 0;
         }
         
+        // Check for existing transaction to prevent duplicates
+        const duplicateCheckQuery = `
+          SELECT COUNT(*) as count FROM fibdash.CSVTransactions 
+          WHERE buchungstag = @buchungstag 
+            AND wertstellung = @wertstellung 
+            AND umsatzart = @umsatzart 
+            AND betrag = @betrag 
+            AND beguenstigter_zahlungspflichtiger = @beguenstigter_zahlungspflichtiger 
+            AND verwendungszweck = @verwendungszweck
+        `;
+        
+        const duplicateCheckResult = await executeQuery(duplicateCheckQuery, {
+          buchungstag: transaction['Buchungstag'] || null,
+          wertstellung: transaction['Valutadatum'] || null,
+          umsatzart: transaction['Buchungstext'] || null,
+          betrag: numericAmount,
+          beguenstigter_zahlungspflichtiger: transaction['Beguenstigter/Zahlungspflichtiger'] || null,
+          verwendungszweck: transaction['Verwendungszweck'] || null
+        });
+        
+        if (duplicateCheckResult.recordset[0].count > 0) {
+          console.log(`Skipping duplicate transaction at row ${i + 1}: ${transaction['Buchungstag']} - ${numericAmount}`);
+          errors.push({
+            row: i + 1,
+            error: 'Duplicate transaction (already exists in database)',
+            transaction: transaction
+          });
+          errorCount++;
+          continue;
+        }
+        
         const insertQuery = `
           INSERT INTO fibdash.CSVTransactions
           (buchungstag, wertstellung, umsatzart, betrag, betrag_original, waehrung,