| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224 |
- import { PrismaClient } from '@prisma/client';
- import { ExcelReaderService } from './excel-reader';
- import { BulkInserter } from './bulk-inserter';
- import { ImportProgressServer } from './websocket-server';
- import { ImportProgress, ImportResult, ProcessedSection } from './types';
- export class ImportProcessor {
- private prisma: PrismaClient;
- private reader: ExcelReaderService;
- private inserter: BulkInserter;
- private progressServer: ImportProgressServer;
- constructor() {
- this.prisma = new PrismaClient();
- this.reader = new ExcelReaderService();
- this.inserter = new BulkInserter();
- this.progressServer = ImportProgressServer.getInstance();
- }
- async processImport(importId: number): Promise<ImportResult> {
- console.log('[IMPORT_PROCESSOR] 🔷 Starting processImport for importId:', importId);
- try {
- // Get import record with layout configuration
- const importRecord = await this.prisma.import.findUnique({
- where: { id: importId },
- include: {
- layout: {
- include: {
- sections: {
- include: { fields: true }
- }
- }
- }
- }
- });
- console.log('[IMPORT_PROCESSOR] 📄 Import record:', {
- id: importRecord?.id,
- name: importRecord?.name,
- hasFileId: !!importRecord?.fileId,
- layoutName: importRecord?.layout?.name,
- sectionsCount: importRecord?.layout?.sections?.length
- });
- if (!importRecord) {
- console.error('[IMPORT_PROCESSOR] ❌ Import not found');
- throw new Error('Import not found');
- }
- if (!importRecord.fileId) {
- console.error('[IMPORT_PROCESSOR] ❌ No file attached to import');
- throw new Error('No file attached to import');
- }
- // Get the actual file
- const fileRecord = await this.prisma.file.findUnique({
- where: { id: importRecord.fileId }
- });
- console.log('[IMPORT_PROCESSOR] 📁 File record:', {
- id: fileRecord?.id,
- filename: fileRecord?.filename,
- size: fileRecord?.size,
- mimetype: fileRecord?.mimetype
- });
- if (!fileRecord) {
- console.error('[IMPORT_PROCESSOR] ❌ File not found');
- throw new Error('File not found');
- }
- // Initialize progress tracking
- const progress: ImportProgress = {
- importId,
- status: 'processing',
- currentSection: '',
- currentRow: 0,
- totalRows: 0,
- errors: [],
- processedSections: 0,
- totalSections: importRecord.layout?.sections?.length ?? 0
- };
- console.log('[IMPORT_PROCESSOR] 📊 Initial progress:', { totalSections: progress.totalSections });
- // Read Excel file
- console.log('[IMPORT_PROCESSOR] 📖 Starting to read Excel file...');
- const sections = await this.reader.readExcelFile(
- fileRecord.data,
- importRecord.layout,
- (sectionProgress: ImportProgress) => {
- this.progressServer.broadcastProgress(importId, sectionProgress);
- }
- );
- console.log('[IMPORT_PROCESSOR] 📖 Excel read complete. Sections parsed:', sections.length);
- sections.forEach((section, idx) => {
- console.log(`[IMPORT_PROCESSOR] Section ${idx + 1}:`, {
- name: section.name,
- tableName: section.tableName,
- dataRows: section.data?.length
- });
- });
- // Process each section
- const processedSections: ProcessedSection[] = [];
- let totalInserted = 0;
- console.log('[IMPORT_PROCESSOR] 🔄 Starting to process', sections.length, 'sections...');
- for (let i = 0; i < sections.length; i++) {
- const section = sections[i];
- if (section) {
- progress.currentSection = section.name ?? '';
- progress.processedSections = i + 1;
- this.progressServer.broadcastProgress(importId, progress);
- console.log(`[IMPORT_PROCESSOR] 📥 Processing section ${i + 1}/${sections.length}:`, {
- name: section.name,
- tableName: section.tableName,
- dataRows: section.data?.length
- });
- try {
- const insertedRows = await this.inserter.insertSectionData(
- section,
- importId,
- (rows: number) => {
- progress.currentRow = rows;
- this.progressServer.broadcastProgress(importId, progress);
- }
- );
- console.log(`[IMPORT_PROCESSOR] ✅ Section ${section.name} inserted ${insertedRows} rows`);
- processedSections.push({
- sectionData: section,
- insertedRows
- });
- totalInserted += insertedRows;
- } catch (error: unknown) {
- const errorMessage = `Error processing section ${section.name ?? 'unknown'}: ${error instanceof Error ? error.message : 'Unknown error'}`;
- console.error('[IMPORT_PROCESSOR] ❌', errorMessage);
- progress.errors.push(errorMessage);
- this.progressServer.broadcastProgress(importId, progress);
- }
- }
- }
- // Import processing completed
- progress.status = 'completed';
- this.progressServer.broadcastProgress(importId, progress);
- console.log('[IMPORT_PROCESSOR] ✅ Import processing completed:', {
- totalInserted,
- sectionsProcessed: processedSections.length,
- errors: progress.errors.length
- });
- return {
- success: true,
- totalInserted,
- sections: processedSections
- };
- } catch (error) {
- // Import processing failed
- const progress: ImportProgress = {
- importId,
- status: 'failed',
- currentSection: '',
- currentRow: 0,
- totalRows: 0,
- errors: [error instanceof Error ? error.message : 'Unknown error'],
- processedSections: 0,
- totalSections: 0
- };
- console.error('[IMPORT_PROCESSOR] ❌ Import processing failed:', error);
- this.progressServer.broadcastProgress(importId, progress);
- return {
- success: false,
- totalInserted: 0,
- sections: [],
- errors: [error instanceof Error ? error.message : 'Unknown error']
- };
- }
- }
- async validateImport(importId: number): Promise<{ valid: boolean; errors: string[] }> {
- const errors: string[] = [];
- try {
- const importRecord = await this.prisma.import.findUnique({
- where: { id: importId },
- include: {
- layout: true
- }
- });
- if (!importRecord) {
- errors.push('Import record not found');
- return { valid: false, errors };
- }
- if (!importRecord.fileId) {
- errors.push('No file attached to import');
- }
- if (!importRecord.layout) {
- errors.push('No layout configuration found');
- }
- return { valid: errors.length === 0, errors };
- } catch (error) {
- errors.push(`Validation error: ${error instanceof Error ? error.message : 'Unknown error'}`);
- return { valid: false, errors };
- }
- }
- }
|