bulk-inserter.ts 4.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143
  1. import { PrismaClient } from '@prisma/client';
  2. import { ReadSectionData } from './types';
  3. export class BulkInserter {
  4. private prisma: PrismaClient;
  5. constructor() {
  6. this.prisma = new PrismaClient();
  7. }
  8. async insertSectionData(
  9. sectionData: ReadSectionData,
  10. importId: number,
  11. onProgress: (rows: number) => void
  12. ): Promise<number> {
  13. const batchSize = 5000;
  14. const totalRows = sectionData.data.length;
  15. let insertedRows = 0;
  16. try {
  17. // Handle specific table names with Prisma models
  18. const tableName = sectionData.tableName;
  19. for (let i = 0; i < totalRows; i += batchSize) {
  20. const batch = sectionData.data.slice(i, i + batchSize);
  21. if (batch.length === 0) continue;
  22. // Prepare data for insertion with proper field mapping
  23. const values = batch.map(row => {
  24. const mappedRow: any = {
  25. importId: importId
  26. };
  27. // Map the row data to match Prisma model field names
  28. Object.keys(row).forEach(key => {
  29. // Convert snake_case to camelCase for Prisma model compatibility
  30. const camelKey = key.replace(/_([a-z])/g, (g) => g[1].toUpperCase());
  31. mappedRow[camelKey] = row[key];
  32. });
  33. return mappedRow;
  34. });
  35. // Use appropriate Prisma model based on table name
  36. if (tableName === 'cintas_install_calendar') {
  37. await this.prisma.cintasInstallCalendar.createMany({
  38. data: values,
  39. skipDuplicates: false
  40. });
  41. } else if (tableName === 'cintas_install_calendar_summary') {
  42. await this.prisma.cintasSummary.createMany({
  43. data: values,
  44. skipDuplicates: false
  45. });
  46. } else {
  47. // Fallback to raw SQL for other tables
  48. await this.prisma.$executeRawUnsafe(
  49. this.buildInsertQuery(tableName, values)
  50. );
  51. }
  52. insertedRows += batch.length;
  53. onProgress(insertedRows);
  54. }
  55. return insertedRows;
  56. } catch (error) {
  57. console.error('Error inserting section data:', error);
  58. throw error;
  59. }
  60. }
  61. private buildInsertQuery(tableName: string, values: any[]): string {
  62. if (values.length === 0) return '';
  63. const keys = Object.keys(values[0]);
  64. const columns = keys.join(', ');
  65. const placeholders = values.map(row => {
  66. const valuesList = keys.map(key => {
  67. const value = row[key];
  68. if (value === null || value === undefined) {
  69. return 'NULL';
  70. }
  71. if (typeof value === 'string') {
  72. return `'${value.replace(/'/g, "''")}'`;
  73. }
  74. return value;
  75. });
  76. return `(${valuesList.join(', ')})`;
  77. }).join(', ');
  78. return `INSERT INTO "${tableName}" (${columns}) VALUES ${placeholders}`;
  79. }
  80. async createImportTable(tableName: string, fields: any[]): Promise<void> {
  81. try {
  82. // Create table if it doesn't exist
  83. const columns = fields.map(field => {
  84. const dataType = this.mapDataType(field.dataType);
  85. return `"${field.importTableColumnName}" ${dataType}`;
  86. }).join(', ');
  87. const createTableQuery = `
  88. CREATE TABLE IF NOT EXISTS "${tableName}" (
  89. id SERIAL PRIMARY KEY,
  90. import_id INTEGER NOT NULL,
  91. ${columns},
  92. created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
  93. )
  94. `;
  95. await this.prisma.$executeRawUnsafe(createTableQuery);
  96. } catch (error) {
  97. console.error('Error creating import table:', error);
  98. throw error;
  99. }
  100. }
  101. private mapDataType(dataType: string): string {
  102. switch (dataType?.toLowerCase()) {
  103. case 'string':
  104. case 'text':
  105. return 'TEXT';
  106. case 'number':
  107. case 'integer':
  108. return 'INTEGER';
  109. case 'decimal':
  110. case 'float':
  111. return 'DECIMAL';
  112. case 'boolean':
  113. return 'BOOLEAN';
  114. case 'date':
  115. return 'DATE';
  116. case 'datetime':
  117. return 'TIMESTAMP';
  118. default:
  119. return 'TEXT';
  120. }
  121. }
  122. }