bulk-inserter.ts 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132
  1. import { PrismaClient } from '@prisma/client';
  2. import { ReadSectionData } from './types';
  3. export class BulkInserter {
  4. private prisma: PrismaClient;
  5. constructor() {
  6. this.prisma = new PrismaClient();
  7. }
  8. async insertSectionData(
  9. sectionData: ReadSectionData,
  10. importId: number,
  11. onProgress: (rows: number) => void
  12. ): Promise<number> {
  13. console.log('[BULK_INSERTER] 🔷 Starting insertSectionData');
  14. console.log('[BULK_INSERTER] 📊 Section:', { name: sectionData.name, tableName: sectionData.tableName, importId, dataRows: sectionData.data?.length });
  15. const batchSize = 5000;
  16. const totalRows = sectionData.data.length;
  17. let insertedRows = 0;
  18. try {
  19. // Handle specific table names with Prisma models
  20. const tableName = sectionData.tableName;
  21. console.log('[BULK_INSERTER] 📋 Table mapping:', tableName);
  22. for (let i = 0; i < totalRows; i += batchSize) {
  23. const batch = sectionData.data.slice(i, i + batchSize);
  24. if (batch.length === 0) continue;
  25. // Prepare data for insertion with proper field mapping
  26. const values = batch.map(row => {
  27. const mappedRow: any = {
  28. importId: importId
  29. };
  30. // Map the row data to match Prisma model field names
  31. Object.keys(row).forEach(key => {
  32. // Convert snake_case to camelCase for Prisma model compatibility
  33. const camelKey = key.replace(/_([a-z0-9])/g, (g) => g[1].toUpperCase());
  34. mappedRow[camelKey] = row[key];
  35. });
  36. return mappedRow;
  37. });
  38. console.log('[BULK_INSERTER] 📤 Batch prepared:', { batchSize: values.length, sampleKeys: Object.keys(values[0] || {}) });
  39. // Use appropriate Prisma model based on table name
  40. if (tableName === 'cintas_install_calendar') {
  41. console.log('[BULK_INSERTER] 📥 Inserting into cintasInstallCalendar...');
  42. await this.prisma.cintasInstallCalendar.createMany({
  43. data: values,
  44. skipDuplicates: false
  45. });
  46. console.log('[BULK_INSERTER] ✅ Inserted into cintasInstallCalendar');
  47. } else if (tableName === 'cintas_install_calendar_summary') {
  48. console.log('[BULK_INSERTER] 📥 Inserting into cintasSummary...');
  49. await this.prisma.cintasSummary.createMany({
  50. data: values,
  51. skipDuplicates: false
  52. });
  53. console.log('[BULK_INSERTER] ✅ Inserted into cintasSummary');
  54. } else if (tableName === 'gow_data') {
  55. console.log('[BULK_INSERTER] 📥 Inserting into gowData...');
  56. await this.prisma.gowData.createMany({
  57. data: values,
  58. skipDuplicates: false
  59. });
  60. console.log('[BULK_INSERTER] ✅ Inserted into gowData');
  61. } else if (tableName === 'gow_fac_id') {
  62. console.log('[BULK_INSERTER] 📥 Inserting into gowFacId...');
  63. await this.prisma.gowFacId.createMany({
  64. data: values,
  65. skipDuplicates: false
  66. });
  67. console.log('[BULK_INSERTER] ✅ Inserted into gowFacId');
  68. } else if (tableName === 'gow_corp_ref') {
  69. console.log('[BULK_INSERTER] 📥 Inserting into gowCorpRef...');
  70. await this.prisma.gowCorpRef.createMany({
  71. data: values,
  72. skipDuplicates: false
  73. });
  74. console.log('[BULK_INSERTER] ✅ Inserted into gowCorpRef');
  75. } else {
  76. console.log('[BULK_INSERTER] 📥 Using raw SQL insert for table:', tableName);
  77. // Fallback to raw SQL for other tables
  78. await this.prisma.$executeRawUnsafe(
  79. this.buildInsertQuery(tableName, values)
  80. );
  81. console.log('[BULK_INSERTER] ✅ Raw SQL insert complete');
  82. }
  83. insertedRows += batch.length;
  84. onProgress(insertedRows);
  85. }
  86. console.log('[BULK_INSERTER] ✅ Section insert complete:', { totalInserted: insertedRows });
  87. return insertedRows;
  88. } catch (error) {
  89. console.error('[BULK_INSERTER] ❌ Error inserting section data:', error);
  90. throw error;
  91. }
  92. }
  93. private buildInsertQuery(tableName: string, values: any[]): string {
  94. if (values.length === 0) return '';
  95. const keys = Object.keys(values[0]);
  96. const columns = keys.map(key => `"${key}"`).join(', ');
  97. const placeholders = values.map(row => {
  98. const valuesList = keys.map(key => {
  99. const value = row[key];
  100. if (value === null || value === undefined) {
  101. return 'NULL';
  102. }
  103. if (typeof value === 'string') {
  104. return `'${value.replace(/'/g, "''")}'`;
  105. }
  106. return value;
  107. });
  108. return `(${valuesList.join(', ')})`;
  109. }).join(', ');
  110. return `INSERT INTO "${tableName}" (${columns}) VALUES ${placeholders}`;
  111. }
  112. }