database-cintas-import-processor.ts 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257
  1. /* eslint-disable @typescript-eslint/no-explicit-any */
  2. import { PrismaClient } from '@prisma/client';
  3. import { DatabaseExcelReaderService } from './database-excel-reader';
  4. import { BulkInserter } from './bulk-inserter';
  5. import { ImportProgressServer } from './websocket-server';
  6. import { ImportProgress, ImportResult } from './types';
  7. export class DatabaseCintasImportProcessor {
  8. private prisma: PrismaClient;
  9. private reader: DatabaseExcelReaderService;
  10. private inserter: BulkInserter;
  11. private progressServer: ImportProgressServer;
  12. constructor() {
  13. this.prisma = new PrismaClient();
  14. this.reader = new DatabaseExcelReaderService();
  15. this.inserter = new BulkInserter();
  16. this.progressServer = ImportProgressServer.getInstance();
  17. }
  18. async processCintasImport(importId: number): Promise<ImportResult> {
  19. try {
  20. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting import processing for ID: ${importId}`);
  21. // Initialize the progress server if not already done
  22. if (!this.progressServer.isServerInitialized()) {
  23. this.progressServer.initialize();
  24. }
  25. // Get import record with layout configuration
  26. const importRecord = await this.prisma.import.findUnique({
  27. where: { id: importId },
  28. include: {
  29. layout: {
  30. include: {
  31. sections: {
  32. include: { fields: true }
  33. }
  34. }
  35. }
  36. }
  37. });
  38. if (!importRecord) {
  39. throw new Error('Import not found');
  40. }
  41. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Loaded import record: ${importRecord.id}`);
  42. // Initialize progress tracking
  43. const progress: ImportProgress = {
  44. importId,
  45. status: 'processing',
  46. currentSection: '',
  47. currentRow: 0,
  48. totalRows: 0,
  49. errors: [],
  50. processedSections: 0,
  51. totalSections: importRecord.layout?.sections?.length || 0
  52. };
  53. // Read Excel file directly from database
  54. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting Excel file reading from database...`);
  55. const sections = await this.reader.readExcelFromDatabase(
  56. importRecord.fileId!,
  57. importRecord.layout,
  58. (sectionProgress) => {
  59. this.progressServer.broadcastProgress(importId, sectionProgress);
  60. }
  61. );
  62. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Excel file read successfully. Found ${sections.length} sections`);
  63. // Process each section
  64. const processedSections = [];
  65. let totalInserted = 0;
  66. for (let i = 0; i < sections.length; i++) {
  67. const section = sections[i];
  68. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Processing section ${i + 1}/${sections.length}: ${section.name}`);
  69. progress.currentSection = section.name;
  70. progress.processedSections = i + 1;
  71. this.progressServer.broadcastProgress(importId, progress);
  72. try {
  73. // Ensure table exists for this section
  74. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Creating table ${section.tableName} for section ${section.name}`);
  75. await this.inserter.createImportTable(section.tableName, section.fields);
  76. const insertedRows = await this.inserter.insertSectionData(
  77. section,
  78. importId,
  79. (rows) => {
  80. progress.currentRow = rows;
  81. this.progressServer.broadcastProgress(importId, progress);
  82. }
  83. );
  84. processedSections.push({
  85. sectionData: section,
  86. insertedRows
  87. });
  88. totalInserted += insertedRows;
  89. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Completed section ${section.name}: ${insertedRows} rows inserted`);
  90. } catch (error) {
  91. const errorMessage = `Error processing section ${section.name}: ${error instanceof Error ? error.message : 'Unknown error'}`;
  92. progress.errors.push(errorMessage);
  93. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: ${errorMessage}`);
  94. this.progressServer.broadcastProgress(importId, progress);
  95. }
  96. }
  97. progress.status = 'completed';
  98. this.progressServer.broadcastProgress(importId, progress);
  99. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Import processing completed successfully. Total inserted: ${totalInserted}`);
  100. return {
  101. success: true,
  102. totalInserted,
  103. sections: processedSections
  104. };
  105. } catch (error) {
  106. const progress: ImportProgress = {
  107. importId,
  108. status: 'failed',
  109. currentSection: '',
  110. currentRow: 0,
  111. totalRows: 0,
  112. errors: [error instanceof Error ? error.message : 'Unknown error'],
  113. processedSections: 0,
  114. totalSections: 0
  115. };
  116. this.progressServer.broadcastProgress(importId, progress);
  117. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: Import processing failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
  118. return {
  119. success: false,
  120. totalInserted: 0,
  121. sections: [],
  122. errors: [error instanceof Error ? error.message : 'Unknown error']
  123. };
  124. }
  125. }
  126. async processCintasImportFromBuffer(
  127. buffer: Buffer,
  128. layoutConfig: any,
  129. onProgress: (progress: ImportProgress) => void
  130. ): Promise<ImportResult> {
  131. try {
  132. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting import processing from buffer`);
  133. // Initialize progress tracking
  134. const progress: ImportProgress = {
  135. importId: 0,
  136. status: 'processing',
  137. currentSection: '',
  138. currentRow: 0,
  139. totalRows: 0,
  140. errors: [],
  141. processedSections: 0,
  142. totalSections: layoutConfig.sections?.length || 0
  143. };
  144. // Read Excel file directly from buffer
  145. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Starting Excel file reading from buffer...`);
  146. const sections = await this.reader.readExcelFromBuffer(
  147. buffer,
  148. layoutConfig,
  149. (sectionProgress) => {
  150. onProgress(sectionProgress);
  151. }
  152. );
  153. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Excel file read successfully. Found ${sections.length} sections`);
  154. // Process each section
  155. const processedSections = [];
  156. let totalInserted = 0;
  157. for (let i = 0; i < sections.length; i++) {
  158. const section = sections[i];
  159. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Processing section ${i + 1}/${sections.length}: ${section.name}`);
  160. progress.currentSection = section.name;
  161. progress.processedSections = i + 1;
  162. onProgress(progress);
  163. try {
  164. // Ensure table exists for this section
  165. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Creating table ${section.tableName} for section ${section.name}`);
  166. await this.inserter.createImportTable(section.tableName, section.fields);
  167. const insertedRows = await this.inserter.insertSectionData(
  168. section,
  169. 0, // Use 0 for buffer-based imports
  170. (rows) => {
  171. progress.currentRow = rows;
  172. onProgress(progress);
  173. }
  174. );
  175. processedSections.push({
  176. sectionData: section,
  177. insertedRows
  178. });
  179. totalInserted += insertedRows;
  180. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Completed section ${section.name}: ${insertedRows} rows inserted`);
  181. } catch (error) {
  182. const errorMessage = `Error processing section ${section.name}: ${error instanceof Error ? error.message : 'Unknown error'}`;
  183. progress.errors.push(errorMessage);
  184. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: ${errorMessage}`);
  185. onProgress(progress);
  186. }
  187. }
  188. console.log(`[${new Date().toISOString()}] [DatabaseCintasImport] Import processing completed successfully. Total inserted: ${totalInserted}`);
  189. return {
  190. success: true,
  191. totalInserted,
  192. sections: processedSections
  193. };
  194. } catch (error) {
  195. const progress: ImportProgress = {
  196. importId: 0,
  197. status: 'failed',
  198. currentSection: '',
  199. currentRow: 0,
  200. totalRows: 0,
  201. errors: [error instanceof Error ? error.message : 'Unknown error'],
  202. processedSections: 0,
  203. totalSections: 0
  204. };
  205. onProgress(progress);
  206. console.error(`[${new Date().toISOString()}] [DatabaseCintasImport] ERROR: Import processing failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
  207. return {
  208. success: false,
  209. totalInserted: 0,
  210. sections: [],
  211. errors: [error instanceof Error ? error.message : 'Unknown error']
  212. };
  213. }
  214. }
  215. }