database-cintas-import-processor.ts 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. import { PrismaClient } from '@prisma/client';
  2. import { DatabaseExcelReaderService } from './database-excel-reader';
  3. import { BulkInserter } from './bulk-inserter';
  4. import { ImportProgressServer } from './websocket-server';
  5. import { ImportProgress, ImportResult } from './types';
  6. export class DatabaseCintasImportProcessor {
  7. private prisma: PrismaClient;
  8. private reader: DatabaseExcelReaderService;
  9. private inserter: BulkInserter;
  10. private progressServer: ImportProgressServer;
  11. constructor() {
  12. this.prisma = new PrismaClient();
  13. this.reader = new DatabaseExcelReaderService();
  14. this.inserter = new BulkInserter();
  15. this.progressServer = ImportProgressServer.getInstance();
  16. }
  17. async processCintasImport(importId: number): Promise<ImportResult> {
  18. try {
  19. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting import processing for ID: ${importId}`);
  20. // Initialize the progress server if not already done
  21. if (!this.progressServer.isServerInitialized()) {
  22. this.progressServer.initialize();
  23. }
  24. // Get import record with layout configuration
  25. const importRecord = await this.prisma.import.findUnique({
  26. where: { id: importId },
  27. include: {
  28. layout: {
  29. include: {
  30. sections: {
  31. include: { fields: true }
  32. }
  33. }
  34. }
  35. }
  36. });
  37. if (!importRecord) {
  38. throw new Error('Import not found');
  39. }
  40. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Loaded import record: ${importRecord.id}`);
  41. // Initialize progress tracking
  42. const progress: ImportProgress = {
  43. importId,
  44. status: 'processing',
  45. currentSection: '',
  46. currentRow: 0,
  47. totalRows: 0,
  48. errors: [],
  49. processedSections: 0,
  50. totalSections: importRecord.layout?.sections?.length || 0
  51. };
  52. // Read Excel file directly from database
  53. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting Excel file reading from database...`);
  54. const sections = await this.reader.readExcelFromDatabase(
  55. importRecord.fileId!,
  56. importRecord.layout,
  57. (sectionProgress) => {
  58. this.progressServer.broadcastProgress(importId, sectionProgress);
  59. }
  60. );
  61. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Excel file read successfully. Found ${sections.length} sections`);
  62. // Process each section
  63. const processedSections = [];
  64. let totalInserted = 0;
  65. for (let i = 0; i < sections.length; i++) {
  66. const section = sections[i];
  67. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Processing section ${i + 1}/${sections.length}: ${section.name}`);
  68. progress.currentSection = section.name;
  69. progress.processedSections = i + 1;
  70. this.progressServer.broadcastProgress(importId, progress);
  71. try {
  72. const insertedRows = await this.inserter.insertSectionData(
  73. section,
  74. importId,
  75. (rows) => {
  76. progress.currentRow = rows;
  77. this.progressServer.broadcastProgress(importId, progress);
  78. }
  79. );
  80. processedSections.push({
  81. sectionData: section,
  82. insertedRows
  83. });
  84. totalInserted += insertedRows;
  85. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Completed section ${section.name}: ${insertedRows} rows inserted`);
  86. } catch (error) {
  87. const errorMessage = `Error processing section ${section.name}: ${error instanceof Error ? error.message : 'Unknown error'}`;
  88. progress.errors.push(errorMessage);
  89. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: ${errorMessage}`);
  90. this.progressServer.broadcastProgress(importId, progress);
  91. }
  92. }
  93. progress.status = 'completed';
  94. this.progressServer.broadcastProgress(importId, progress);
  95. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Import processing completed successfully. Total inserted: ${totalInserted}`);
  96. return {
  97. success: true,
  98. totalInserted,
  99. sections: processedSections
  100. };
  101. } catch (error) {
  102. const progress: ImportProgress = {
  103. importId,
  104. status: 'failed',
  105. currentSection: '',
  106. currentRow: 0,
  107. totalRows: 0,
  108. errors: [error instanceof Error ? error.message : 'Unknown error'],
  109. processedSections: 0,
  110. totalSections: 0
  111. };
  112. this.progressServer.broadcastProgress(importId, progress);
  113. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: Import processing failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
  114. return {
  115. success: false,
  116. totalInserted: 0,
  117. sections: [],
  118. errors: [error instanceof Error ? error.message : 'Unknown error']
  119. };
  120. }
  121. }
  122. async processCintasImportFromBuffer(
  123. buffer: Buffer,
  124. layoutConfig: any,
  125. onProgress: (progress: ImportProgress) => void
  126. ): Promise<ImportResult> {
  127. try {
  128. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting import processing from buffer`);
  129. // Initialize progress tracking
  130. const progress: ImportProgress = {
  131. importId: 0,
  132. status: 'processing',
  133. currentSection: '',
  134. currentRow: 0,
  135. totalRows: 0,
  136. errors: [],
  137. processedSections: 0,
  138. totalSections: layoutConfig.sections?.length || 0
  139. };
  140. // Read Excel file directly from buffer
  141. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting Excel file reading from buffer...`);
  142. const sections = await this.reader.readExcelFromBuffer(
  143. buffer,
  144. layoutConfig,
  145. (sectionProgress) => {
  146. onProgress(sectionProgress);
  147. }
  148. );
  149. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Excel file read successfully. Found ${sections.length} sections`);
  150. // Process each section
  151. const processedSections = [];
  152. let totalInserted = 0;
  153. for (let i = 0; i < sections.length; i++) {
  154. const section = sections[i];
  155. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Processing section ${i + 1}/${sections.length}: ${section.name}`);
  156. progress.currentSection = section.name;
  157. progress.processedSections = i + 1;
  158. onProgress(progress);
  159. try {
  160. const insertedRows = await this.inserter.insertSectionData(
  161. section,
  162. 0, // Use 0 for buffer-based imports
  163. (rows) => {
  164. progress.currentRow = rows;
  165. onProgress(progress);
  166. }
  167. );
  168. processedSections.push({
  169. sectionData: section,
  170. insertedRows
  171. });
  172. totalInserted += insertedRows;
  173. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Completed section ${section.name}: ${insertedRows} rows inserted`);
  174. } catch (error) {
  175. const errorMessage = `Error processing section ${section.name}: ${error instanceof Error ? error.message : 'Unknown error'}`;
  176. progress.errors.push(errorMessage);
  177. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: ${errorMessage}`);
  178. onProgress(progress);
  179. }
  180. }
  181. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Import processing completed successfully. Total inserted: ${totalInserted}`);
  182. return {
  183. success: true,
  184. totalInserted,
  185. sections: processedSections
  186. };
  187. } catch (error) {
  188. const progress: ImportProgress = {
  189. importId: 0,
  190. status: 'failed',
  191. currentSection: '',
  192. currentRow: 0,
  193. totalRows: 0,
  194. errors: [error instanceof Error ? error.message : 'Unknown error'],
  195. processedSections: 0,
  196. totalSections: 0
  197. };
  198. onProgress(progress);
  199. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: Import processing failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
  200. return {
  201. success: false,
  202. totalInserted: 0,
  203. sections: [],
  204. errors: [error instanceof Error ? error.message : 'Unknown error']
  205. };
  206. }
  207. }
  208. }