bulk-inserter.ts 3.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122
  1. import { PrismaClient } from '@prisma/client';
  2. import { ReadSectionData } from './types';
  3. export class BulkInserter {
  4. private prisma: PrismaClient;
  5. constructor() {
  6. this.prisma = new PrismaClient();
  7. }
  8. async insertSectionData(
  9. sectionData: ReadSectionData,
  10. importId: number,
  11. onProgress: (rows: number) => void
  12. ): Promise<number> {
  13. const batchSize = 5000;
  14. const totalRows = sectionData.data.length;
  15. let insertedRows = 0;
  16. try {
  17. // Create table name safely
  18. const tableName = sectionData.tableName;
  19. for (let i = 0; i < totalRows; i += batchSize) {
  20. const batch = sectionData.data.slice(i, i + batchSize);
  21. if (batch.length === 0) continue;
  22. // Prepare data for insertion
  23. const values = batch.map(row => ({
  24. import_id: importId,
  25. ...row
  26. }));
  27. // Use Prisma's createMany for batch insertion
  28. // Note: This assumes the table has a corresponding Prisma model
  29. // For dynamic table names, we would need to use raw SQL
  30. const result = await this.prisma.$executeRawUnsafe(
  31. this.buildInsertQuery(tableName, values)
  32. );
  33. insertedRows += batch.length;
  34. onProgress(insertedRows);
  35. }
  36. return insertedRows;
  37. } catch (error) {
  38. console.error('Error inserting section data:', error);
  39. throw error;
  40. }
  41. }
  42. private buildInsertQuery(tableName: string, values: any[]): string {
  43. if (values.length === 0) return '';
  44. const keys = Object.keys(values[0]);
  45. const columns = keys.join(', ');
  46. const placeholders = values.map(row => {
  47. const valuesList = keys.map(key => {
  48. const value = row[key];
  49. if (value === null || value === undefined) {
  50. return 'NULL';
  51. }
  52. if (typeof value === 'string') {
  53. return `'${value.replace(/'/g, "''")}'`;
  54. }
  55. return value;
  56. });
  57. return `(${valuesList.join(', ')})`;
  58. }).join(', ');
  59. return `INSERT INTO "${tableName}" (${columns}) VALUES ${placeholders}`;
  60. }
  61. async createImportTable(tableName: string, fields: any[]): Promise<void> {
  62. try {
  63. // Create table if it doesn't exist
  64. const columns = fields.map(field => {
  65. const dataType = this.mapDataType(field.dataType);
  66. return `"${field.importTableColumnName}" ${dataType}`;
  67. }).join(', ');
  68. const createTableQuery = `
  69. CREATE TABLE IF NOT EXISTS "${tableName}" (
  70. id SERIAL PRIMARY KEY,
  71. import_id INTEGER NOT NULL,
  72. ${columns},
  73. created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
  74. )
  75. `;
  76. await this.prisma.$executeRawUnsafe(createTableQuery);
  77. } catch (error) {
  78. console.error('Error creating import table:', error);
  79. throw error;
  80. }
  81. }
  82. private mapDataType(dataType: string): string {
  83. switch (dataType?.toLowerCase()) {
  84. case 'string':
  85. case 'text':
  86. return 'TEXT';
  87. case 'number':
  88. case 'integer':
  89. return 'INTEGER';
  90. case 'decimal':
  91. case 'float':
  92. return 'DECIMAL';
  93. case 'boolean':
  94. return 'BOOLEAN';
  95. case 'date':
  96. return 'DATE';
  97. case 'datetime':
  98. return 'TIMESTAMP';
  99. default:
  100. return 'TEXT';
  101. }
  102. }
  103. }