bulk-inserter.ts 4.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144
  1. /* eslint-disable @typescript-eslint/no-explicit-any */
  2. import { PrismaClient } from '@prisma/client';
  3. import { ReadSectionData } from './types';
  4. export class BulkInserter {
  5. private prisma: PrismaClient;
  6. constructor() {
  7. this.prisma = new PrismaClient();
  8. }
  9. async insertSectionData(
  10. sectionData: ReadSectionData,
  11. importId: number,
  12. onProgress: (rows: number) => void
  13. ): Promise<number> {
  14. const batchSize = 5000;
  15. const totalRows = sectionData.data.length;
  16. let insertedRows = 0;
  17. try {
  18. // Handle specific table names with Prisma models
  19. const tableName = sectionData.tableName;
  20. for (let i = 0; i < totalRows; i += batchSize) {
  21. const batch = sectionData.data.slice(i, i + batchSize);
  22. if (batch.length === 0) continue;
  23. // Prepare data for insertion with proper field mapping
  24. const values = batch.map(row => {
  25. const mappedRow: any = {
  26. importId: importId
  27. };
  28. // Map the row data to match Prisma model field names
  29. Object.keys(row).forEach(key => {
  30. // Convert snake_case to camelCase for Prisma model compatibility
  31. const camelKey = key.replace(/_([a-z])/g, (g) => g[1].toUpperCase());
  32. mappedRow[camelKey] = row[key];
  33. });
  34. return mappedRow;
  35. });
  36. // Use appropriate Prisma model based on table name
  37. if (tableName === 'cintas_install_calendar') {
  38. await this.prisma.cintasInstallCalendar.createMany({
  39. data: values,
  40. skipDuplicates: false
  41. });
  42. } else if (tableName === 'cintas_install_calendar_summary') {
  43. await this.prisma.cintasSummary.createMany({
  44. data: values,
  45. skipDuplicates: false
  46. });
  47. } else {
  48. // Fallback to raw SQL for other tables
  49. await this.prisma.$executeRawUnsafe(
  50. this.buildInsertQuery(tableName, values)
  51. );
  52. }
  53. insertedRows += batch.length;
  54. onProgress(insertedRows);
  55. }
  56. return insertedRows;
  57. } catch (error) {
  58. console.error('Error inserting section data:', error);
  59. throw error;
  60. }
  61. }
  62. private buildInsertQuery(tableName: string, values: any[]): string {
  63. if (values.length === 0) return '';
  64. const keys = Object.keys(values[0]);
  65. const columns = keys.join(', ');
  66. const placeholders = values.map(row => {
  67. const valuesList = keys.map(key => {
  68. const value = row[key];
  69. if (value === null || value === undefined) {
  70. return 'NULL';
  71. }
  72. if (typeof value === 'string') {
  73. return `'${value.replace(/'/g, "''")}'`;
  74. }
  75. return value;
  76. });
  77. return `(${valuesList.join(', ')})`;
  78. }).join(', ');
  79. return `INSERT INTO "${tableName}" (${columns}) VALUES ${placeholders}`;
  80. }
  81. async createImportTable(tableName: string, fields: any[]): Promise<void> {
  82. try {
  83. // Create table if it doesn't exist
  84. const columns = fields.map(field => {
  85. const dataType = this.mapDataType(field.dataType);
  86. return `"${field.importTableColumnName}" ${dataType}`;
  87. }).join(', ');
  88. const createTableQuery = `
  89. CREATE TABLE IF NOT EXISTS "${tableName}" (
  90. id SERIAL PRIMARY KEY,
  91. import_id INTEGER NOT NULL,
  92. ${columns},
  93. created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
  94. )
  95. `;
  96. await this.prisma.$executeRawUnsafe(createTableQuery);
  97. } catch (error) {
  98. console.error('Error creating import table:', error);
  99. throw error;
  100. }
  101. }
  102. private mapDataType(dataType: string): string {
  103. switch (dataType?.toLowerCase()) {
  104. case 'string':
  105. case 'text':
  106. return 'TEXT';
  107. case 'number':
  108. case 'integer':
  109. return 'INTEGER';
  110. case 'decimal':
  111. case 'float':
  112. return 'DECIMAL';
  113. case 'boolean':
  114. return 'BOOLEAN';
  115. case 'date':
  116. return 'DATE';
  117. case 'datetime':
  118. return 'TIMESTAMP';
  119. default:
  120. return 'TEXT';
  121. }
  122. }
  123. }