bulk-inserter.ts 3.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114
  1. import { PrismaClient } from '@prisma/client';
  2. import { ReadSectionData } from './types';
  3. export class BulkInserter {
  4. private prisma: PrismaClient;
  5. constructor() {
  6. this.prisma = new PrismaClient();
  7. }
  8. async insertSectionData(
  9. sectionData: ReadSectionData,
  10. importId: number,
  11. onProgress: (rows: number) => void
  12. ): Promise<number> {
  13. const batchSize = 5000;
  14. const totalRows = sectionData.data.length;
  15. let insertedRows = 0;
  16. try {
  17. // Handle specific table names with Prisma models
  18. const tableName = sectionData.tableName;
  19. for (let i = 0; i < totalRows; i += batchSize) {
  20. const batch = sectionData.data.slice(i, i + batchSize);
  21. if (batch.length === 0) continue;
  22. // Prepare data for insertion with proper field mapping
  23. const values = batch.map(row => {
  24. const mappedRow: any = {
  25. importId: importId
  26. };
  27. // Map the row data to match Prisma model field names
  28. Object.keys(row).forEach(key => {
  29. // Convert snake_case to camelCase for Prisma model compatibility
  30. const camelKey = key.replace(/_([a-z])/g, (g) => g[1].toUpperCase());
  31. mappedRow[camelKey] = row[key];
  32. });
  33. return mappedRow;
  34. });
  35. console.log(batch)
  36. console.log(values);
  37. // Use appropriate Prisma model based on table name
  38. if (tableName === 'cintas_install_calendar') {
  39. await this.prisma.cintasInstallCalendar.createMany({
  40. data: values,
  41. skipDuplicates: false
  42. });
  43. } else if (tableName === 'cintas_install_calendar_summary') {
  44. await this.prisma.cintasSummary.createMany({
  45. data: values,
  46. skipDuplicates: false
  47. });
  48. } else if (tableName === 'gow_data') {
  49. await this.prisma.gowData.createMany({
  50. data: values,
  51. skipDuplicates: false
  52. });
  53. } else if (tableName === 'gow_fac_id') {
  54. await this.prisma.gowFacId.createMany({
  55. data: values,
  56. skipDuplicates: false
  57. });
  58. } else if (tableName === 'gow_corp_ref') {
  59. await this.prisma.gowCorpRef.createMany({
  60. data: values,
  61. skipDuplicates: false
  62. });
  63. } else {
  64. // Fallback to raw SQL for other tables
  65. await this.prisma.$executeRawUnsafe(
  66. this.buildInsertQuery(tableName, values)
  67. );
  68. }
  69. insertedRows += batch.length;
  70. onProgress(insertedRows);
  71. }
  72. return insertedRows;
  73. } catch (error) {
  74. console.error('Error inserting section data:', error);
  75. throw error;
  76. }
  77. }
  78. private buildInsertQuery(tableName: string, values: any[]): string {
  79. if (values.length === 0) return '';
  80. const keys = Object.keys(values[0]);
  81. const columns = keys.map(key => `"${key}"`).join(', ');
  82. const placeholders = values.map(row => {
  83. const valuesList = keys.map(key => {
  84. const value = row[key];
  85. if (value === null || value === undefined) {
  86. return 'NULL';
  87. }
  88. if (typeof value === 'string') {
  89. return `'${value.replace(/'/g, "''")}'`;
  90. }
  91. return value;
  92. });
  93. return `(${valuesList.join(', ')})`;
  94. }).join(', ');
  95. return `INSERT INTO "${tableName}" (${columns}) VALUES ${placeholders}`;
  96. }
  97. }