database-cintas-import-processor.ts 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269
  1. /* eslint-disable @typescript-eslint/no-explicit-any */
  2. import { PrismaClient } from '@prisma/client';
  3. import { DatabaseExcelReaderService } from './database-excel-reader';
  4. import { BulkInserter } from './bulk-inserter';
  5. import { ImportProgressServer } from './websocket-server';
  6. import { ImportProgress, ImportResult } from './types';
  7. export class DatabaseCintasImportProcessor {
  8. private prisma: PrismaClient;
  9. private reader: DatabaseExcelReaderService;
  10. private inserter: BulkInserter;
  11. private progressServer: ImportProgressServer;
  12. constructor() {
  13. this.prisma = new PrismaClient();
  14. this.reader = new DatabaseExcelReaderService();
  15. this.inserter = new BulkInserter();
  16. this.progressServer = ImportProgressServer.getInstance();
  17. }
  18. async processCintasImport(importId: number): Promise<ImportResult> {
  19. try {
  20. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting import processing for ID: ${importId}`);
  21. // Initialize the progress server if not already done
  22. if (!this.progressServer.isServerInitialized()) {
  23. this.progressServer.initialize();
  24. }
  25. // Get import record with layout configuration
  26. const importRecord = await this.prisma.import.findUnique({
  27. where: { id: importId },
  28. include: {
  29. layout: {
  30. include: {
  31. sections: {
  32. include: { fields: true }
  33. }
  34. }
  35. }
  36. }
  37. });
  38. if (!importRecord) {
  39. throw new Error('Import not found');
  40. }
  41. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Loaded import record: ${importRecord.id}`);
  42. // Initialize progress tracking
  43. const progress: ImportProgress = {
  44. importId,
  45. status: 'processing',
  46. currentSection: '',
  47. currentRow: 0,
  48. totalRows: 0,
  49. errors: [],
  50. processedSections: 0,
  51. totalSections: importRecord.layout?.sections?.length || 0
  52. };
  53. // Read Excel file directly from database
  54. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting Excel file reading from database...`);
  55. const sections = await this.reader.readExcelFromDatabase(
  56. importRecord.fileId!,
  57. importRecord.layout,
  58. (sectionProgress) => {
  59. this.progressServer.broadcastProgress(importId, sectionProgress);
  60. }
  61. );
  62. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Excel file read successfully. Found ${sections.length} sections`);
  63. // Process each section
  64. const processedSections = [];
  65. let totalInserted = 0;
  66. for (let i = 0; i < sections.length; i++) {
  67. const section = sections[i];
  68. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Processing section ${i+1}/${sections.length}: ${section.name}`);
  69. progress.currentSection = section.name;
  70. progress.processedSections = i + 1;
  71. this.progressServer.broadcastProgress(importId, progress);
  72. try {
  73. // Ensure table exists for this section
  74. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Creating table ${section.tableName} for section ${section.name}`);
  75. await this.inserter.createImportTable(section.tableName, section.fields);
  76. const insertedRows = await this.inserter.insertSectionData(
  77. section,
  78. importId,
  79. (rows) => {
  80. progress.currentRow = rows;
  81. this.progressServer.broadcastProgress(importId, progress);
  82. }
  83. );
  84. processedSections.push({
  85. sectionData: section,
  86. insertedRows
  87. });
  88. totalInserted += insertedRows;
  89. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Completed section ${section.name}: ${insertedRows} rows inserted`);
  90. } catch (error) {
  91. const errorMessage = `Error processing section ${section.name}: ${error instanceof Error ? error.message : 'Unknown error'}`;
  92. progress.errors.push(errorMessage);
  93. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: ${errorMessage}`);
  94. this.progressServer.broadcastProgress(importId, progress);
  95. }
  96. }
  97. // Run the stored procedure to calculate summary
  98. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Running summary calculation procedure...`);
  99. try {
  100. await this.prisma.$executeRawUnsafe(
  101. `CALL cintas_calculate_summary(${importId})`
  102. );
  103. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Summary calculation completed successfully`);
  104. } catch (error) {
  105. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: Summary calculation failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
  106. progress.errors.push(`Stored procedure error: ${error instanceof Error ? error.message : 'Unknown error'}`);
  107. }
  108. progress.status = 'completed';
  109. this.progressServer.broadcastProgress(importId, progress);
  110. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Import processing completed successfully. Total inserted: ${totalInserted}`);
  111. return {
  112. success: true,
  113. totalInserted,
  114. sections: processedSections
  115. };
  116. } catch (error) {
  117. const progress: ImportProgress = {
  118. importId,
  119. status: 'failed',
  120. currentSection: '',
  121. currentRow: 0,
  122. totalRows: 0,
  123. errors: [error instanceof Error ? error.message : 'Unknown error'],
  124. processedSections: 0,
  125. totalSections: 0
  126. };
  127. this.progressServer.broadcastProgress(importId, progress);
  128. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: Import processing failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
  129. return {
  130. success: false,
  131. totalInserted: 0,
  132. sections: [],
  133. errors: [error instanceof Error ? error.message : 'Unknown error']
  134. };
  135. }
  136. }
  137. async processCintasImportFromBuffer(
  138. buffer: Buffer,
  139. layoutConfig: any,
  140. onProgress: (progress: ImportProgress) => void
  141. ): Promise<ImportResult> {
  142. try {
  143. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting import processing from buffer`);
  144. // Initialize progress tracking
  145. const progress: ImportProgress = {
  146. importId: 0,
  147. status: 'processing',
  148. currentSection: '',
  149. currentRow: 0,
  150. totalRows: 0,
  151. errors: [],
  152. processedSections: 0,
  153. totalSections: layoutConfig.sections?.length || 0
  154. };
  155. // Read Excel file directly from buffer
  156. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting Excel file reading from buffer...`);
  157. const sections = await this.reader.readExcelFromBuffer(
  158. buffer,
  159. layoutConfig,
  160. (sectionProgress) => {
  161. onProgress(sectionProgress);
  162. }
  163. );
  164. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Excel file read successfully. Found ${sections.length} sections`);
  165. // Process each section
  166. const processedSections = [];
  167. let totalInserted = 0;
  168. for (let i = 0; i < sections.length; i++) {
  169. const section = sections[i];
  170. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Processing section ${i+1}/${sections.length}: ${section.name}`);
  171. progress.currentSection = section.name;
  172. progress.processedSections = i + 1;
  173. onProgress(progress);
  174. try {
  175. // Ensure table exists for this section
  176. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Creating table ${section.tableName} for section ${section.name}`);
  177. await this.inserter.createImportTable(section.tableName, section.fields);
  178. const insertedRows = await this.inserter.insertSectionData(
  179. section,
  180. 0, // Use 0 for buffer-based imports
  181. (rows) => {
  182. progress.currentRow = rows;
  183. onProgress(progress);
  184. }
  185. );
  186. processedSections.push({
  187. sectionData: section,
  188. insertedRows
  189. });
  190. totalInserted += insertedRows;
  191. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Completed section ${section.name}: ${insertedRows} rows inserted`);
  192. } catch (error) {
  193. const errorMessage = `Error processing section ${section.name}: ${error instanceof Error ? error.message : 'Unknown error'}`;
  194. progress.errors.push(errorMessage);
  195. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: ${errorMessage}`);
  196. onProgress(progress);
  197. }
  198. }
  199. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Import processing completed successfully. Total inserted: ${totalInserted}`);
  200. return {
  201. success: true,
  202. totalInserted,
  203. sections: processedSections
  204. };
  205. } catch (error) {
  206. const progress: ImportProgress = {
  207. importId: 0,
  208. status: 'failed',
  209. currentSection: '',
  210. currentRow: 0,
  211. totalRows: 0,
  212. errors: [error instanceof Error ? error.message : 'Unknown error'],
  213. processedSections: 0,
  214. totalSections: 0
  215. };
  216. onProgress(progress);
  217. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: Import processing failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
  218. return {
  219. success: false,
  220. totalInserted: 0,
  221. sections: [],
  222. errors: [error instanceof Error ? error.message : 'Unknown error']
  223. };
  224. }
  225. }
  226. }