database-cintas-import-processor.ts 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256
  1. import { PrismaClient } from '@prisma/client';
  2. import { DatabaseExcelReaderService } from './database-excel-reader';
  3. import { BulkInserter } from './bulk-inserter';
  4. import { ImportProgressServer } from './websocket-server';
  5. import { ImportProgress, ImportResult } from './types';
  6. export class DatabaseCintasImportProcessor {
  7. private prisma: PrismaClient;
  8. private reader: DatabaseExcelReaderService;
  9. private inserter: BulkInserter;
  10. private progressServer: ImportProgressServer;
  11. constructor() {
  12. this.prisma = new PrismaClient();
  13. this.reader = new DatabaseExcelReaderService();
  14. this.inserter = new BulkInserter();
  15. this.progressServer = ImportProgressServer.getInstance();
  16. }
  17. async processCintasImport(importId: number): Promise<ImportResult> {
  18. try {
  19. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting import processing for ID: ${importId}`);
  20. // Initialize the progress server if not already done
  21. if (!this.progressServer.isServerInitialized()) {
  22. this.progressServer.initialize();
  23. }
  24. // Get import record with layout configuration
  25. const importRecord = await this.prisma.import.findUnique({
  26. where: { id: importId },
  27. include: {
  28. layout: {
  29. include: {
  30. sections: {
  31. include: { fields: true }
  32. }
  33. }
  34. }
  35. }
  36. });
  37. if (!importRecord) {
  38. throw new Error('Import not found');
  39. }
  40. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Loaded import record: ${importRecord.id}`);
  41. // Initialize progress tracking
  42. const progress: ImportProgress = {
  43. importId,
  44. status: 'processing',
  45. currentSection: '',
  46. currentRow: 0,
  47. totalRows: 0,
  48. errors: [],
  49. processedSections: 0,
  50. totalSections: importRecord.layout?.sections?.length || 0
  51. };
  52. // Read Excel file directly from database
  53. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting Excel file reading from database...`);
  54. const sections = await this.reader.readExcelFromDatabase(
  55. importRecord.fileId!,
  56. importRecord.layout,
  57. (sectionProgress) => {
  58. this.progressServer.broadcastProgress(importId, sectionProgress);
  59. }
  60. );
  61. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Excel file read successfully. Found ${sections.length} sections`);
  62. // Process each section
  63. const processedSections = [];
  64. let totalInserted = 0;
  65. for (let i = 0; i < sections.length; i++) {
  66. const section = sections[i];
  67. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Processing section ${i + 1}/${sections.length}: ${section.name}`);
  68. progress.currentSection = section.name;
  69. progress.processedSections = i + 1;
  70. this.progressServer.broadcastProgress(importId, progress);
  71. try {
  72. // Ensure table exists for this section
  73. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Creating table ${section.tableName} for section ${section.name}`);
  74. await this.inserter.createImportTable(section.tableName, section.fields);
  75. const insertedRows = await this.inserter.insertSectionData(
  76. section,
  77. importId,
  78. (rows) => {
  79. progress.currentRow = rows;
  80. this.progressServer.broadcastProgress(importId, progress);
  81. }
  82. );
  83. processedSections.push({
  84. sectionData: section,
  85. insertedRows
  86. });
  87. totalInserted += insertedRows;
  88. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Completed section ${section.name}: ${insertedRows} rows inserted`);
  89. } catch (error) {
  90. const errorMessage = `Error processing section ${section.name}: ${error instanceof Error ? error.message : 'Unknown error'}`;
  91. progress.errors.push(errorMessage);
  92. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: ${errorMessage}`);
  93. this.progressServer.broadcastProgress(importId, progress);
  94. }
  95. }
  96. progress.status = 'completed';
  97. this.progressServer.broadcastProgress(importId, progress);
  98. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Import processing completed successfully. Total inserted: ${totalInserted}`);
  99. return {
  100. success: true,
  101. totalInserted,
  102. sections: processedSections
  103. };
  104. } catch (error) {
  105. const progress: ImportProgress = {
  106. importId,
  107. status: 'failed',
  108. currentSection: '',
  109. currentRow: 0,
  110. totalRows: 0,
  111. errors: [error instanceof Error ? error.message : 'Unknown error'],
  112. processedSections: 0,
  113. totalSections: 0
  114. };
  115. this.progressServer.broadcastProgress(importId, progress);
  116. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: Import processing failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
  117. return {
  118. success: false,
  119. totalInserted: 0,
  120. sections: [],
  121. errors: [error instanceof Error ? error.message : 'Unknown error']
  122. };
  123. }
  124. }
  125. async processCintasImportFromBuffer(
  126. buffer: Buffer,
  127. layoutConfig: any,
  128. onProgress: (progress: ImportProgress) => void
  129. ): Promise<ImportResult> {
  130. try {
  131. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting import processing from buffer`);
  132. // Initialize progress tracking
  133. const progress: ImportProgress = {
  134. importId: 0,
  135. status: 'processing',
  136. currentSection: '',
  137. currentRow: 0,
  138. totalRows: 0,
  139. errors: [],
  140. processedSections: 0,
  141. totalSections: layoutConfig.sections?.length || 0
  142. };
  143. // Read Excel file directly from buffer
  144. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting Excel file reading from buffer...`);
  145. const sections = await this.reader.readExcelFromBuffer(
  146. buffer,
  147. layoutConfig,
  148. (sectionProgress) => {
  149. onProgress(sectionProgress);
  150. }
  151. );
  152. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Excel file read successfully. Found ${sections.length} sections`);
  153. // Process each section
  154. const processedSections = [];
  155. let totalInserted = 0;
  156. for (let i = 0; i < sections.length; i++) {
  157. const section = sections[i];
  158. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Processing section ${i + 1}/${sections.length}: ${section.name}`);
  159. progress.currentSection = section.name;
  160. progress.processedSections = i + 1;
  161. onProgress(progress);
  162. try {
  163. // Ensure table exists for this section
  164. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Creating table ${section.tableName} for section ${section.name}`);
  165. await this.inserter.createImportTable(section.tableName, section.fields);
  166. const insertedRows = await this.inserter.insertSectionData(
  167. section,
  168. 0, // Use 0 for buffer-based imports
  169. (rows) => {
  170. progress.currentRow = rows;
  171. onProgress(progress);
  172. }
  173. );
  174. processedSections.push({
  175. sectionData: section,
  176. insertedRows
  177. });
  178. totalInserted += insertedRows;
  179. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Completed section ${section.name}: ${insertedRows} rows inserted`);
  180. } catch (error) {
  181. const errorMessage = `Error processing section ${section.name}: ${error instanceof Error ? error.message : 'Unknown error'}`;
  182. progress.errors.push(errorMessage);
  183. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: ${errorMessage}`);
  184. onProgress(progress);
  185. }
  186. }
  187. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Import processing completed successfully. Total inserted: ${totalInserted}`);
  188. return {
  189. success: true,
  190. totalInserted,
  191. sections: processedSections
  192. };
  193. } catch (error) {
  194. const progress: ImportProgress = {
  195. importId: 0,
  196. status: 'failed',
  197. currentSection: '',
  198. currentRow: 0,
  199. totalRows: 0,
  200. errors: [error instanceof Error ? error.message : 'Unknown error'],
  201. processedSections: 0,
  202. totalSections: 0
  203. };
  204. onProgress(progress);
  205. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: Import processing failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
  206. return {
  207. success: false,
  208. totalInserted: 0,
  209. sections: [],
  210. errors: [error instanceof Error ? error.message : 'Unknown error']
  211. };
  212. }
  213. }
  214. }