bulk-inserter.ts 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161
  1. import { PrismaClient } from '@prisma/client';
  2. import { ReadSectionData } from './types';
  3. export class BulkInserter {
  4. private prisma: PrismaClient;
  5. constructor() {
  6. this.prisma = new PrismaClient();
  7. }
  8. async insertSectionData(
  9. sectionData: ReadSectionData,
  10. importId: number,
  11. onProgress: (rows: number) => void
  12. ): Promise<number> {
  13. const batchSize = 5000;
  14. const totalRows = sectionData.data.length;
  15. let insertedRows = 0;
  16. try {
  17. // Handle specific table names with Prisma models
  18. const tableName = sectionData.tableName;
  19. for (let i = 0; i < totalRows; i += batchSize) {
  20. const batch = sectionData.data.slice(i, i + batchSize);
  21. if (batch.length === 0) continue;
  22. // Prepare data for insertion with proper field mapping
  23. const values = batch.map(row => {
  24. const mappedRow: any = {
  25. importId: importId,
  26. createdAt: new Date(),
  27. updatedAt: new Date()
  28. };
  29. // Map the row data to match Prisma model field names
  30. Object.keys(row).forEach(key => {
  31. // Convert snake_case to camelCase for Prisma model compatibility
  32. // Also handle digits (e.g., corp_id_2 -> corpId2)
  33. const camelKey = key.replace(/_([a-z0-9])/g, (g) => g[1].toUpperCase());
  34. mappedRow[camelKey] = row[key];
  35. });
  36. return mappedRow;
  37. });
  38. // Use appropriate Prisma model based on table name
  39. if (tableName === 'cintas_install_calendar') {
  40. await this.prisma.cintasInstallCalendar.createMany({
  41. data: values,
  42. skipDuplicates: false
  43. });
  44. } else if (tableName === 'cintas_install_calendar_summary' || tableName === 'cintas_intall_calendar_summary') {
  45. await this.prisma.cintasSummary.createMany({
  46. data: values,
  47. skipDuplicates: false
  48. });
  49. } else if (tableName === 'gow_data') {
  50. await this.prisma.gowData.createMany({
  51. data: values,
  52. skipDuplicates: false
  53. });
  54. } else if (tableName === 'gow_fac_id') {
  55. await this.prisma.gowFacId.createMany({
  56. data: values,
  57. skipDuplicates: false
  58. });
  59. } else if (tableName === 'gow_corp_ref') {
  60. await this.prisma.gowCorpRef.createMany({
  61. data: values,
  62. skipDuplicates: false
  63. });
  64. } else {
  65. // Fallback to raw SQL for other tables
  66. await this.prisma.$executeRawUnsafe(
  67. this.buildInsertQuery(tableName, values)
  68. );
  69. }
  70. insertedRows += batch.length;
  71. onProgress(insertedRows);
  72. }
  73. return insertedRows;
  74. } catch (error) {
  75. console.error('Error inserting section data:', error);
  76. throw error;
  77. }
  78. }
  79. private buildInsertQuery(tableName: string, values: any[]): string {
  80. if (values.length === 0) return '';
  81. const keys = Object.keys(values[0]);
  82. const columns = keys.map(key => `"${key}"`).join(', ');
  83. const placeholders = values.map(row => {
  84. const valuesList = keys.map(key => {
  85. const value = row[key];
  86. if (value === null || value === undefined) {
  87. return 'NULL';
  88. }
  89. if (typeof value === 'string') {
  90. return `'${value.replace(/'/g, "''")}'`;
  91. }
  92. return value;
  93. });
  94. return `(${valuesList.join(', ')})`;
  95. }).join(', ');
  96. return `INSERT INTO "${tableName}" (${columns}) VALUES ${placeholders}`;
  97. }
  98. async createImportTable(tableName: string, fields: any[]): Promise<void> {
  99. try {
  100. // Create table if it doesn't exist
  101. const columns = fields.map(field => {
  102. const dataType = this.mapDataType(field.dataType);
  103. return `"${field.importTableColumnName}" ${dataType}`;
  104. }).join(', ');
  105. const createTableQuery = `
  106. CREATE TABLE IF NOT EXISTS "${tableName}" (
  107. id SERIAL PRIMARY KEY,
  108. "importId" INTEGER NOT NULL,
  109. ${columns},
  110. "createdAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP
  111. )
  112. `;
  113. await this.prisma.$executeRawUnsafe(createTableQuery);
  114. } catch (error) {
  115. console.error('Error creating import table:', error);
  116. throw error;
  117. }
  118. }
  119. private mapDataType(dataType: string): string {
  120. switch (dataType?.toLowerCase()) {
  121. case 'string':
  122. case 'text':
  123. return 'TEXT';
  124. case 'number':
  125. case 'integer':
  126. return 'INTEGER';
  127. case 'decimal':
  128. case 'float':
  129. return 'DECIMAL';
  130. case 'boolean':
  131. return 'BOOLEAN';
  132. case 'date':
  133. return 'DATE';
  134. case 'datetime':
  135. return 'TIMESTAMP';
  136. default:
  137. return 'TEXT';
  138. }
  139. }
  140. }