feat(webhooks): add script to retry failed webhooks and log results

Introduce a new script to process and retry failed webhooks stored in a designated directory. The script reads JSON files containing webhook payloads and error information, attempts to resend the payloads, and logs the success or failure of each retry. Additionally, it creates a processed directory to organize successfully retried webhooks. This enhances error handling and recovery for webhook failures.

.cursor/rules/restart.mdc

@@ -0,0 +1,4 @@
+---
+alwaysApply: true
+---
+the server is always running an can be restarted with pm2 restart 3

retry_failed_webhooks.js

@@ -0,0 +1,93 @@
+#!/usr/bin/env node
+// Script to retry failed webhooks after code fixes
+const fs = require('fs');
+const path = require('path');
+const axios = require('axios');
+
+const failedWebhooksDir = path.join(__dirname, 'failed_webhooks');
+const WEBHOOK_URL = 'http://localhost:9304/releasehook_kjfhdkf987987';
+
+async function retryFailedWebhook(filepath) {
+  try {
+    console.log(`\nProcessing: ${path.basename(filepath)}`);
+    
+    const data = JSON.parse(fs.readFileSync(filepath, 'utf8'));
+    const payload = data.payload;
+    
+    console.log(`Repository: ${payload?.repository?.name || 'unknown'}`);
+    console.log(`Original error: ${data.error}`);
+    console.log(`Failed at: ${data.timestamp}`);
+    
+    // Retry the webhook
+    const response = await axios.post(WEBHOOK_URL, payload, {
+      headers: {
+        'Content-Type': 'application/json'
+      },
+      timeout: 30000
+    });
+    
+    console.log(`✅ Retry successful: ${response.status}`);
+    
+    // Move the file to processed directory
+    const processedDir = path.join(failedWebhooksDir, 'processed');
+    if (!fs.existsSync(processedDir)) {
+      fs.mkdirSync(processedDir);
+    }
+    
+    const newPath = path.join(processedDir, path.basename(filepath));
+    fs.renameSync(filepath, newPath);
+    console.log(`Moved to processed: ${path.basename(newPath)}`);
+    
+    return true;
+  } catch (error) {
+    console.log(`❌ Retry failed: ${error.message}`);
+    if (error.response) {
+      console.log(`Response status: ${error.response.status}`);
+      console.log(`Response data:`, error.response.data);
+    }
+    return false;
+  }
+}
+
+async function main() {
+  if (!fs.existsSync(failedWebhooksDir)) {
+    console.log('No failed_webhooks directory found.');
+    return;
+  }
+  
+  const files = fs.readdirSync(failedWebhooksDir)
+    .filter(file => file.endsWith('.json'))
+    .map(file => path.join(failedWebhooksDir, file));
+  
+  if (files.length === 0) {
+    console.log('No failed webhooks to retry.');
+    return;
+  }
+  
+  console.log(`Found ${files.length} failed webhook(s) to retry:`);
+  
+  let successCount = 0;
+  let failCount = 0;
+  
+  for (const filepath of files) {
+    const success = await retryFailedWebhook(filepath);
+    if (success) {
+      successCount++;
+    } else {
+      failCount++;
+    }
+    
+    // Add a small delay between retries
+    await new Promise(resolve => setTimeout(resolve, 1000));
+  }
+  
+  console.log(`\n📊 Summary:`);
+  console.log(`✅ Successful retries: ${successCount}`);
+  console.log(`❌ Failed retries: ${failCount}`);
+}
+
+if (require.main === module) {
+  main().catch(console.error);
+}
+
+module.exports = { retryFailedWebhook, main };

x.js

@@ -229,6 +229,12 @@ if (!fs.existsSync(logsDir)) {
   fs.mkdirSync(logsDir);
 }
 
+// Create failed webhooks directory if it doesn't exist
+const failedWebhooksDir = path.join(__dirname, 'failed_webhooks');
+if (!fs.existsSync(failedWebhooksDir)) {
+  fs.mkdirSync(failedWebhooksDir);
+}
+
 // Logger function
 function logMessage(message, type = 'info') {
   const timestamp = new Date().toISOString();
@@ -241,6 +247,29 @@ function logMessage(message, type = 'info') {
   fs.appendFileSync(logFile, logEntry + '\n');
 }
 
+// Function to dump failed webhooks for retry
+function dumpFailedWebhook(payload, error, reason = 'unknown') {
+  try {
+    const timestamp = new Date().toISOString();
+    const filename = `failed_webhook_${timestamp.replace(/[:.]/g, '-')}_${payload?.repository?.name || 'unknown'}.json`;
+    const filepath = path.join(failedWebhooksDir, filename);
+    
+    const failedWebhookData = {
+      timestamp,
+      reason,
+      error: error?.message || String(error),
+      payload
+    };
+    
+    fs.writeFileSync(filepath, JSON.stringify(failedWebhookData, null, 2));
+    logMessage(`Failed webhook dumped to: ${filename}`, 'info');
+    return filepath;
+  } catch (dumpError) {
+    logMessage(`Failed to dump webhook: ${dumpError.message}`, 'error');
+    return null;
+  }
+}
+
 // Telegram bot function
 async function sendTelegramMessage(message) {
   const botToken = process.env.TELEGRAM_BOT_TOKEN;
@@ -267,6 +296,8 @@ async function sendTelegramMessage(message) {
     if (error.response) {
       logMessage(`Telegram API error: ${JSON.stringify(error.response.data)}`, 'error');
     }
+    // Re-throw the error so it can be caught by the calling code
+    throw error;
   }
 }
 
@@ -393,6 +424,7 @@ function formatCommitMessage(payload) {
       }
     } catch (e) {
       // already logged inside summarizer; keep silent here
+      logMessage(`Executive summary generation failed: ${e.message}`, 'warn');
     }
 
     // Ensure each section starts on its own line; compare may be plain URL text.
@@ -419,6 +451,7 @@ app.post('/releasehook_kjfhdkf987987', (req, res) => {
       .then((telegramMessage) => sendTelegramMessage(telegramMessage))
       .catch(error => {
         logMessage(`Error sending Telegram message: ${error.message}`, 'error');
+        dumpFailedWebhook(payload, error, 'telegram_send_error');
       });
 
     // Set a flag to track if we've sent a response