seb/fibdash
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add OpenAI API integration and document processing features

Browse Source 
- Added OpenAI API key configuration to .env.example.
- Integrated OpenAI for document processing, including markdown conversion and data extraction.
- Implemented new API routes for fetching document processing status and handling various processing tasks.
- Enhanced the App component to manage document status and processing states with user feedback via Snackbar.
- Updated CSVImportPanel and TableManagement components to support navigation to specific tabs based on processing results.
- Introduced transaction handling in the database configuration for improved error management during document processing.
This commit is contained in:
sebseb7
2025-08-06 11:11:23 +02:00
parent d60da0a7aa
commit 281754de22
12 changed files with 790 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

263
client/src/App.js
View File

@@ -1,11 +1,16 @@
import React, { Component } from 'react';
import { ThemeProvider, createTheme } from '@mui/material/styles';
import CssBaseline from '@mui/material/CssBaseline';
import { Container, AppBar, Toolbar, Typography, Button, Box, Tabs, Tab } from '@mui/material';
import { Container, AppBar, Toolbar, Typography, Button, Box, Tabs, Tab, Badge, Chip, Divider, Snackbar, Alert } from '@mui/material';
import LoginIcon from '@mui/icons-material/Login';
import DashboardIcon from '@mui/icons-material/Dashboard';
import DownloadIcon from '@mui/icons-material/Download';
import TableChart from '@mui/icons-material/TableChart';
import PlayArrowIcon from '@mui/icons-material/PlayArrow';
import DocumentScannerIcon from '@mui/icons-material/DocumentScanner';
import ExtractIcon from '@mui/icons-material/TextSnippet';
import EmailIcon from '@mui/icons-material/Email';
import UploadIcon from '@mui/icons-material/Upload';
import AuthService from './services/AuthService';
import DataViewer from './components/DataViewer';
import Login from './components/Login';
@@ -31,6 +36,18 @@ class App extends Component {
loading: true,
exportData: null, // { selectedMonth, canExport, onExport }
currentView: 'dashboard', // 'dashboard' or 'tables'
documentStatus: null,
processingStatus: {
markdown: false,
extraction: false,
datevSync: false,
datevUpload: false
},
snackbar: {
open: false,
message: '',
severity: 'info' // 'success', 'error', 'warning', 'info'
}
};
this.authService = new AuthService();
}
@@ -39,6 +56,15 @@ class App extends Component {
this.checkAuthStatus();
}
componentDidUpdate(prevState) {
// Clear targetTab after navigation is complete
if (this.state.targetTab && prevState.currentView !== this.state.currentView) {
setTimeout(() => {
this.setState({ targetTab: null });
}, 100); // Small delay to ensure navigation completes
}
}
checkAuthStatus = async () => {
try {
const token = localStorage.getItem('token');
@@ -46,6 +72,7 @@ class App extends Component {
const user = await this.authService.verifyToken(token);
if (user) {
this.setState({ isAuthenticated: true, user, loading: false });
this.fetchDocumentStatus();
return;
}
}
@@ -62,6 +89,7 @@ class App extends Component {
if (result.success) {
localStorage.setItem('token', result.token);
this.setState({ isAuthenticated: true, user: result.user });
this.fetchDocumentStatus();
}
} catch (error) {
console.error('Login failed:', error);
@@ -83,8 +111,131 @@ class App extends Component {
this.setState({ currentView: newValue });
};
showSnackbar = (message, severity = 'info') => {
this.setState({
snackbar: {
open: true,
message,
severity
}
});
};
handleSnackbarClose = (event, reason) => {
if (reason === 'clickaway') {
return;
}
this.setState({
snackbar: {
...this.state.snackbar,
open: false
}
});
};
fetchDocumentStatus = async () => {
try {
const token = localStorage.getItem('token');
if (!token) {
console.log('No token found for document status');
return;
}
console.log('Fetching document status...');
const response = await fetch('/api/data/document-status', {
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
});
if (response.ok) {
const status = await response.json();
console.log('Document status received:', status);
this.setState({ documentStatus: status });
} else {
console.error('Failed to fetch document status:', response.status, await response.text());
}
} catch (error) {
console.error('Error fetching document status:', error);
}
};
handleProcessing = async (processType) => {
if (this.state.processingStatus[processType]) {
return; // Already processing
}
// Handle datev upload navigation
if (processType === 'datev-upload') {
this.setState({
currentView: 'tables',
targetTab: {
level1: 3, // CSV Import tab
level2: 'DATEV_LINKS' // DATEV Beleglinks tab
}
});
return;
}
// Check if there are documents to process
const statusKey = processType === 'datev-sync' ? 'needDatevSync' :
processType === 'extraction' ? 'needExtraction' : 'needMarkdown';
if (!this.state.documentStatus || this.state.documentStatus[statusKey] === 0) {
this.showSnackbar(`No documents need ${processType} processing at this time.`, 'info');
return;
}
this.setState(prevState => ({
processingStatus: {
...prevState.processingStatus,
[processType]: true
}
}));
try {
const token = localStorage.getItem('token');
if (!token) return;
const response = await fetch(`/api/data/process-${processType}`, {
method: 'POST',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
});
if (response.ok) {
const result = await response.json();
console.log(`${processType} processing result:`, result);
this.showSnackbar(`${processType} processing completed successfully!`, 'success');
// Refresh document status after successful processing
await this.fetchDocumentStatus();
} else {
const error = await response.json();
console.error(`Failed to process ${processType}:`, error);
this.showSnackbar(`Failed to process ${processType}: ${error.error || response.status}`, 'error');
}
} catch (error) {
console.error(`Error processing ${processType}:`, error);
this.showSnackbar(`Error processing ${processType}: ${error.message}`, 'error');
} finally {
this.setState(prevState => ({
processingStatus: {
...prevState.processingStatus,
[processType]: false
}
}));
}
};
render() {
const { isAuthenticated, user, loading, currentView } = this.state;
const { isAuthenticated, user, loading, currentView, documentStatus, processingStatus, snackbar } = this.state;
// Debug logging
console.log('App render - documentStatus:', documentStatus);
console.log('App render - isAuthenticated:', isAuthenticated);
if (loading) {
return (
@@ -149,6 +300,97 @@ class App extends Component {
sx={{ minHeight: 48 }}
/>
</Tabs>
<Divider orientation="vertical" flexItem sx={{ mx: 2, backgroundColor: 'rgba(255, 255, 255, 0.3)' }} />
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1 }}>
<Button
color="inherit"
size="small"
onClick={() => this.handleProcessing('markdown')}
disabled={processingStatus.markdown || !documentStatus}
sx={{
minWidth: 'auto',
px: 1,
'&:hover': { backgroundColor: 'rgba(255, 255, 255, 0.1)' }
}}
title="Process markdown conversion"
>
<Badge
badgeContent={documentStatus?.needMarkdown || 0}
color={documentStatus?.needMarkdown > 0 ? "error" : "default"}
max={999999}
sx={{ mr: 0.5 }}
>
<DocumentScannerIcon fontSize="small" />
</Badge>
{processingStatus.markdown && <PlayArrowIcon fontSize="small" />}
</Button>
<Button
color="inherit"
size="small"
onClick={() => this.handleProcessing('extraction')}
disabled={processingStatus.extraction || !documentStatus}
sx={{
minWidth: 'auto',
px: 1,
'&:hover': { backgroundColor: 'rgba(255, 255, 255, 0.1)' }
}}
title="Process data extraction"
>
<Badge
badgeContent={documentStatus?.needExtraction || 0}
color={documentStatus?.needExtraction > 0 ? "warning" : "default"}
max={999999}
sx={{ mr: 0.5 }}
>
<ExtractIcon fontSize="small" />
</Badge>
{processingStatus.extraction && <PlayArrowIcon fontSize="small" />}
</Button>
<Button
color="inherit"
size="small"
onClick={() => this.handleProcessing('datev-sync')}
disabled={processingStatus.datevSync || !documentStatus}
sx={{
minWidth: 'auto',
px: 1,
'&:hover': { backgroundColor: 'rgba(255, 255, 255, 0.1)' }
}}
title="Process Datev sync"
>
<Badge
badgeContent={documentStatus?.needDatevSync || 0}
color={documentStatus?.needDatevSync > 0 ? "info" : "default"}
max={999999}
sx={{ mr: 0.5 }}
>
<EmailIcon fontSize="small" />
</Badge>
{processingStatus.datevSync && <PlayArrowIcon fontSize="small" />}
</Button>
<Button
color="inherit"
size="small"
onClick={() => this.handleProcessing('datev-upload')}
disabled={processingStatus.datevUpload || !documentStatus}
sx={{
minWidth: 'auto',
px: 1,
'&:hover': { backgroundColor: 'rgba(255, 255, 255, 0.1)' }
}}
title="Process Datev CSV upload"
>
<Badge
badgeContent={documentStatus?.needDatevUpload || 0}
color={documentStatus?.needDatevUpload > 0 ? "secondary" : "default"}
max={999999}
sx={{ mr: 0.5 }}
>
<UploadIcon fontSize="small" />
</Badge>
{processingStatus.datevUpload && <PlayArrowIcon fontSize="small" />}
</Button>
</Box>
{this.state.exportData && (
<Button
color="inherit"
@@ -194,12 +436,29 @@ class App extends Component {
onUpdateExportData={this.updateExportData}
currentView={currentView}
onViewChange={this.handleViewChange}
targetTab={this.state.targetTab}
/>
) : (
<Login onLogin={this.handleLogin} />
)}
</Container>
</Box>
<Snackbar
open={snackbar.open}
autoHideDuration={6000}
onClose={this.handleSnackbarClose}
anchorOrigin={{ vertical: 'bottom', horizontal: 'right' }}
>
<Alert
onClose={this.handleSnackbarClose}
severity={snackbar.severity}
variant="filled"
sx={{ width: '100%' }}
>
{snackbar.message}
</Alert>
</Snackbar>
</ThemeProvider>
);
}

15
client/src/components/CSVImportDialog.js
View File

@@ -56,6 +56,20 @@ class CSVImportPanel extends Component {
this.datevFileInputRef = React.createRef();
}
componentDidMount() {
// Check if we should navigate to a specific tab
if (this.props.targetTab) {
this.setState({ activeTab: this.props.targetTab });
}
}
componentDidUpdate(prevProps) {
// Handle targetTab changes
if (this.props.targetTab !== prevProps.targetTab && this.props.targetTab) {
this.setState({ activeTab: this.props.targetTab });
}
}
// Tab switch resets type-specific state but keeps success state as-is
handleTabChange = (_e, value) => {
this.setState({
@@ -344,6 +358,7 @@ class CSVImportPanel extends Component {
)}
{currentHeaders && (
<Box sx={{ mb: 2 }}>
<Typography variant="subtitle2" gutterBottom>
Erkannte Spalten ({currentHeaders.length}):

2
client/src/components/DataViewer.js
View File

@@ -165,7 +165,7 @@ class DataViewer extends Component {
</>
) : (
<Box sx={{ flex: 1, minHeight: 0, overflow: 'auto', p: 2 }}>
<TableManagement user={user} />
<TableManagement user={user} targetTab={this.props.targetTab} />
</Box>
)}
</Box>

16
client/src/components/TableManagement.js
View File

@@ -26,6 +26,21 @@ class TableManagement extends Component {
};
}
componentDidMount() {
// Check if we should navigate to a specific tab
if (this.props.targetTab?.level1 !== undefined) {
this.setState({ activeTab: this.props.targetTab.level1 });
}
}
componentDidUpdate(prevProps) {
// Handle targetTab changes
if (this.props.targetTab?.level1 !== prevProps.targetTab?.level1 &&
this.props.targetTab?.level1 !== undefined) {
this.setState({ activeTab: this.props.targetTab.level1 });
}
}
handleTabChange = (event, newValue) => {
this.setState({ activeTab: newValue });
};
@@ -92,6 +107,7 @@ class TableManagement extends Component {
</Typography>
<CSVImportPanel
user={user}
targetTab={this.props.targetTab?.level2}
/>
</Box>
)}