import-processor.ts 6.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224
  1. import { PrismaClient } from '@prisma/client';
  2. import { ExcelReaderService } from './excel-reader';
  3. import { BulkInserter } from './bulk-inserter';
  4. import { ImportProgressServer } from './websocket-server';
  5. import { ImportProgress, ImportResult, ProcessedSection } from './types';
  6. export class ImportProcessor {
  7. private prisma: PrismaClient;
  8. private reader: ExcelReaderService;
  9. private inserter: BulkInserter;
  10. private progressServer: ImportProgressServer;
  11. constructor() {
  12. this.prisma = new PrismaClient();
  13. this.reader = new ExcelReaderService();
  14. this.inserter = new BulkInserter();
  15. this.progressServer = ImportProgressServer.getInstance();
  16. }
  17. async processImport(importId: number): Promise<ImportResult> {
  18. console.log('[IMPORT_PROCESSOR] 🔷 Starting processImport for importId:', importId);
  19. try {
  20. // Get import record with layout configuration
  21. const importRecord = await this.prisma.import.findUnique({
  22. where: { id: importId },
  23. include: {
  24. layout: {
  25. include: {
  26. sections: {
  27. include: { fields: true }
  28. }
  29. }
  30. }
  31. }
  32. });
  33. console.log('[IMPORT_PROCESSOR] 📄 Import record:', {
  34. id: importRecord?.id,
  35. name: importRecord?.name,
  36. hasFileId: !!importRecord?.fileId,
  37. layoutName: importRecord?.layout?.name,
  38. sectionsCount: importRecord?.layout?.sections?.length
  39. });
  40. if (!importRecord) {
  41. console.error('[IMPORT_PROCESSOR] ❌ Import not found');
  42. throw new Error('Import not found');
  43. }
  44. if (!importRecord.fileId) {
  45. console.error('[IMPORT_PROCESSOR] ❌ No file attached to import');
  46. throw new Error('No file attached to import');
  47. }
  48. // Get the actual file
  49. const fileRecord = await this.prisma.file.findUnique({
  50. where: { id: importRecord.fileId }
  51. });
  52. console.log('[IMPORT_PROCESSOR] 📁 File record:', {
  53. id: fileRecord?.id,
  54. filename: fileRecord?.filename,
  55. size: fileRecord?.size,
  56. mimetype: fileRecord?.mimetype
  57. });
  58. if (!fileRecord) {
  59. console.error('[IMPORT_PROCESSOR] ❌ File not found');
  60. throw new Error('File not found');
  61. }
  62. // Initialize progress tracking
  63. const progress: ImportProgress = {
  64. importId,
  65. status: 'processing',
  66. currentSection: '',
  67. currentRow: 0,
  68. totalRows: 0,
  69. errors: [],
  70. processedSections: 0,
  71. totalSections: importRecord.layout?.sections?.length ?? 0
  72. };
  73. console.log('[IMPORT_PROCESSOR] 📊 Initial progress:', { totalSections: progress.totalSections });
  74. // Read Excel file
  75. console.log('[IMPORT_PROCESSOR] 📖 Starting to read Excel file...');
  76. const sections = await this.reader.readExcelFile(
  77. fileRecord.data,
  78. importRecord.layout,
  79. (sectionProgress: ImportProgress) => {
  80. this.progressServer.broadcastProgress(importId, sectionProgress);
  81. }
  82. );
  83. console.log('[IMPORT_PROCESSOR] 📖 Excel read complete. Sections parsed:', sections.length);
  84. sections.forEach((section, idx) => {
  85. console.log(`[IMPORT_PROCESSOR] Section ${idx + 1}:`, {
  86. name: section.name,
  87. tableName: section.tableName,
  88. dataRows: section.data?.length
  89. });
  90. });
  91. // Process each section
  92. const processedSections: ProcessedSection[] = [];
  93. let totalInserted = 0;
  94. console.log('[IMPORT_PROCESSOR] 🔄 Starting to process', sections.length, 'sections...');
  95. for (let i = 0; i < sections.length; i++) {
  96. const section = sections[i];
  97. if (section) {
  98. progress.currentSection = section.name ?? '';
  99. progress.processedSections = i + 1;
  100. this.progressServer.broadcastProgress(importId, progress);
  101. console.log(`[IMPORT_PROCESSOR] 📥 Processing section ${i + 1}/${sections.length}:`, {
  102. name: section.name,
  103. tableName: section.tableName,
  104. dataRows: section.data?.length
  105. });
  106. try {
  107. const insertedRows = await this.inserter.insertSectionData(
  108. section,
  109. importId,
  110. (rows: number) => {
  111. progress.currentRow = rows;
  112. this.progressServer.broadcastProgress(importId, progress);
  113. }
  114. );
  115. console.log(`[IMPORT_PROCESSOR] ✅ Section ${section.name} inserted ${insertedRows} rows`);
  116. processedSections.push({
  117. sectionData: section,
  118. insertedRows
  119. });
  120. totalInserted += insertedRows;
  121. } catch (error: unknown) {
  122. const errorMessage = `Error processing section ${section.name ?? 'unknown'}: ${error instanceof Error ? error.message : 'Unknown error'}`;
  123. console.error('[IMPORT_PROCESSOR] ❌', errorMessage);
  124. progress.errors.push(errorMessage);
  125. this.progressServer.broadcastProgress(importId, progress);
  126. }
  127. }
  128. }
  129. // Import processing completed
  130. progress.status = 'completed';
  131. this.progressServer.broadcastProgress(importId, progress);
  132. console.log('[IMPORT_PROCESSOR] ✅ Import processing completed:', {
  133. totalInserted,
  134. sectionsProcessed: processedSections.length,
  135. errors: progress.errors.length
  136. });
  137. return {
  138. success: true,
  139. totalInserted,
  140. sections: processedSections
  141. };
  142. } catch (error) {
  143. // Import processing failed
  144. const progress: ImportProgress = {
  145. importId,
  146. status: 'failed',
  147. currentSection: '',
  148. currentRow: 0,
  149. totalRows: 0,
  150. errors: [error instanceof Error ? error.message : 'Unknown error'],
  151. processedSections: 0,
  152. totalSections: 0
  153. };
  154. console.error('[IMPORT_PROCESSOR] ❌ Import processing failed:', error);
  155. this.progressServer.broadcastProgress(importId, progress);
  156. return {
  157. success: false,
  158. totalInserted: 0,
  159. sections: [],
  160. errors: [error instanceof Error ? error.message : 'Unknown error']
  161. };
  162. }
  163. }
  164. async validateImport(importId: number): Promise<{ valid: boolean; errors: string[] }> {
  165. const errors: string[] = [];
  166. try {
  167. const importRecord = await this.prisma.import.findUnique({
  168. where: { id: importId },
  169. include: {
  170. layout: true
  171. }
  172. });
  173. if (!importRecord) {
  174. errors.push('Import record not found');
  175. return { valid: false, errors };
  176. }
  177. if (!importRecord.fileId) {
  178. errors.push('No file attached to import');
  179. }
  180. if (!importRecord.layout) {
  181. errors.push('No layout configuration found');
  182. }
  183. return { valid: errors.length === 0, errors };
  184. } catch (error) {
  185. errors.push(`Validation error: ${error instanceof Error ? error.message : 'Unknown error'}`);
  186. return { valid: false, errors };
  187. }
  188. }
  189. }