bulk-inserter.ts 3.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123
  1. /* eslint-disable @typescript-eslint/no-explicit-any */
  2. import { PrismaClient } from '@prisma/client';
  3. import { ReadSectionData } from './types';
  4. export class BulkInserter {
  5. private prisma: PrismaClient;
  6. constructor() {
  7. this.prisma = new PrismaClient();
  8. }
  9. async insertSectionData(
  10. sectionData: ReadSectionData,
  11. importId: number,
  12. onProgress: (rows: number) => void
  13. ): Promise<number> {
  14. const batchSize = 5000;
  15. const totalRows = sectionData.data.length;
  16. let insertedRows = 0;
  17. try {
  18. // Create table name safely
  19. const tableName = sectionData.tableName;
  20. for (let i = 0; i < totalRows; i += batchSize) {
  21. const batch = sectionData.data.slice(i, i + batchSize);
  22. if (batch.length === 0) continue;
  23. // Prepare data for insertion
  24. const values = batch.map(row => ({
  25. import_id: importId,
  26. ...row
  27. }));
  28. // Use Prisma's createMany for batch insertion
  29. // Note: This assumes the table has a corresponding Prisma model
  30. // For dynamic table names, we would need to use raw SQL
  31. await this.prisma.$executeRawUnsafe(
  32. this.buildInsertQuery(tableName, values)
  33. );
  34. insertedRows += batch.length;
  35. onProgress(insertedRows);
  36. }
  37. return insertedRows;
  38. } catch (error) {
  39. console.error('Error inserting section data:', error);
  40. throw error;
  41. }
  42. }
  43. private buildInsertQuery(tableName: string, values: any[]): string {
  44. if (values.length === 0) return '';
  45. const keys = Object.keys(values[0]);
  46. const columns = keys.join(', ');
  47. const placeholders = values.map(row => {
  48. const valuesList = keys.map(key => {
  49. const value = row[key];
  50. if (value === null || value === undefined) {
  51. return 'NULL';
  52. }
  53. if (typeof value === 'string') {
  54. return `'${value.replace(/'/g, "''")}'`;
  55. }
  56. return value;
  57. });
  58. return `(${valuesList.join(', ')})`;
  59. }).join(', ');
  60. return `INSERT INTO "${tableName}" (${columns}) VALUES ${placeholders}`;
  61. }
  62. async createImportTable(tableName: string, fields: any[]): Promise<void> {
  63. try {
  64. // Create table if it doesn't exist
  65. const columns = fields.map(field => {
  66. const dataType = this.mapDataType(field.dataType);
  67. return `"${field.importTableColumnName}" ${dataType}`;
  68. }).join(', ');
  69. const createTableQuery = `
  70. CREATE TABLE IF NOT EXISTS "${tableName}" (
  71. id SERIAL PRIMARY KEY,
  72. import_id INTEGER NOT NULL,
  73. ${columns},
  74. created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
  75. )
  76. `;
  77. await this.prisma.$executeRawUnsafe(createTableQuery);
  78. } catch (error) {
  79. console.error('Error creating import table:', error);
  80. throw error;
  81. }
  82. }
  83. private mapDataType(dataType: string): string {
  84. switch (dataType?.toLowerCase()) {
  85. case 'string':
  86. case 'text':
  87. return 'TEXT';
  88. case 'number':
  89. case 'integer':
  90. return 'INTEGER';
  91. case 'decimal':
  92. case 'float':
  93. return 'DECIMAL';
  94. case 'boolean':
  95. return 'BOOLEAN';
  96. case 'date':
  97. return 'DATE';
  98. case 'datetime':
  99. return 'TIMESTAMP';
  100. default:
  101. return 'TEXT';
  102. }
  103. }
  104. }