WIP

feat(excel-import): add comprehensive Excel import processing system with WebSocket progress tracking

- Implement ExcelReaderService for reading and parsing Excel files with configurable layouts
- Add BulkInserter class for efficient bulk data insertion with batch processing
- Create ImportProcessor to orchestrate the entire import workflow
- Add WebSocket server for real-time progress updates during imports
- Include TypeScript interfaces matching C# models from Excelerator codebase
- Support for grid and properties section types with field mapping
- Add progress tracking with section-by-section processing updates
- Implement error handling and validation throughout import pipeline

app/api/imports/[id]/progress/route.ts

@@ -0,0 +1,22 @@
+import { NextRequest } from 'next/server';
+
+// This is a WebSocket endpoint for real-time import progress updates
+export async function GET(
+  request: NextRequest,
+  { params }: { params: { id: string } }
+) {
+  // This will be handled by the WebSocket upgrade
+  return new Response('WebSocket endpoint for import progress', { status: 101 });
+}
+
+// WebSocket upgrade handler
+export async function upgradeWebSocket(
+  request: NextRequest,
+  { params }: { params: { id: string } }
+) {
+  const importId = parseInt(params.id);
+  
+  // This would typically be handled by a custom server setup
+  // For now, we'll use the standalone WebSocket server
+  return new Response(null, { status: 101 });
+}

app/lib/excel-import/bulk-inserter.ts

@@ -0,0 +1,122 @@
+import { PrismaClient } from '@prisma/client';
+import { ReadSectionData } from './types';
+
+export class BulkInserter {
+  private prisma: PrismaClient;
+
+  constructor() {
+    this.prisma = new PrismaClient();
+  }
+
+  async insertSectionData(
+    sectionData: ReadSectionData,
+    importId: number,
+    onProgress: (rows: number) => void
+  ): Promise<number> {
+    const batchSize = 5000;
+    const totalRows = sectionData.data.length;
+    let insertedRows = 0;
+
+    try {
+      // Create table name safely
+      const tableName = sectionData.tableName;
+      
+      for (let i = 0; i < totalRows; i += batchSize) {
+        const batch = sectionData.data.slice(i, i + batchSize);
+        
+        if (batch.length === 0) continue;
+
+        // Prepare data for insertion
+        const values = batch.map(row => ({
+          import_id: importId,
+          ...row
+        }));
+
+        // Use Prisma's createMany for batch insertion
+        // Note: This assumes the table has a corresponding Prisma model
+        // For dynamic table names, we would need to use raw SQL
+        const result = await this.prisma.$executeRawUnsafe(
+          this.buildInsertQuery(tableName, values)
+        );
+
+        insertedRows += batch.length;
+        onProgress(insertedRows);
+      }
+
+      return insertedRows;
+
+    } catch (error) {
+      console.error('Error inserting section data:', error);
+      throw error;
+    }
+  }
+
+  private buildInsertQuery(tableName: string, values: any[]): string {
+    if (values.length === 0) return '';
+    
+    const keys = Object.keys(values[0]);
+    const columns = keys.join(', ');
+    
+    const placeholders = values.map(row => {
+      const valuesList = keys.map(key => {
+        const value = row[key];
+        if (value === null || value === undefined) {
+          return 'NULL';
+        }
+        if (typeof value === 'string') {
+          return `'${value.replace(/'/g, "''")}'`;
+        }
+        return value;
+      });
+      return `(${valuesList.join(', ')})`;
+    }).join(', ');
+
+    return `INSERT INTO "${tableName}" (${columns}) VALUES ${placeholders}`;
+  }
+
+  async createImportTable(tableName: string, fields: any[]): Promise<void> {
+    try {
+      // Create table if it doesn't exist
+      const columns = fields.map(field => {
+        const dataType = this.mapDataType(field.dataType);
+        return `"${field.importTableColumnName}" ${dataType}`;
+      }).join(', ');
+
+      const createTableQuery = `
+        CREATE TABLE IF NOT EXISTS "${tableName}" (
+          id SERIAL PRIMARY KEY,
+          import_id INTEGER NOT NULL,
+          ${columns},
+          created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+      `;
+
+      await this.prisma.$executeRawUnsafe(createTableQuery);
+    } catch (error) {
+      console.error('Error creating import table:', error);
+      throw error;
+    }
+  }
+
+  private mapDataType(dataType: string): string {
+    switch (dataType?.toLowerCase()) {
+      case 'string':
+      case 'text':
+        return 'TEXT';
+      case 'number':
+      case 'integer':
+        return 'INTEGER';
+      case 'decimal':
+      case 'float':
+        return 'DECIMAL';
+      case 'boolean':
+        return 'BOOLEAN';
+      case 'date':
+        return 'DATE';
+      case 'datetime':
+        return 'TIMESTAMP';
+      default:
+        return 'TEXT';
+    }
+  }
+}

app/lib/excel-import/excel-reader.ts

@@ -0,0 +1,209 @@
+import * as ExcelJS from 'exceljs';
+import { ReadSectionData, LayoutSectionField, SectionTypeEnum, FieldTypeEnum, ImportProgress } from './types';
+
+export class ExcelReaderService {
+  async readExcelFile(
+    fileBuffer: Buffer,
+    layoutConfig: any,
+    onProgress: (progress: ImportProgress) => void
+  ): Promise<ReadSectionData[]> {
+    const workbook = new ExcelJS.Workbook();
+    await workbook.xlsx.load(fileBuffer);
+    
+    const results: ReadSectionData[] = [];
+    const totalSections = layoutConfig.sections?.length || 0;
+    
+    // Initialize progress
+    onProgress({
+      importId: 0, // Will be set by caller
+      status: 'processing',
+      currentSection: '',
+      currentRow: 0,
+      totalRows: 0,
+      errors: [],
+      processedSections: 0,
+      totalSections
+    });
+
+    for (let sectionIndex = 0; sectionIndex < totalSections; sectionIndex++) {
+      const section = layoutConfig.sections[sectionIndex];
+      const worksheet = workbook.getWorksheet(section.sheetName);
+      
+      if (!worksheet) {
+        onProgress({
+          importId: 0,
+          status: 'processing',
+          currentSection: section.name,
+          currentRow: 0,
+          totalRows: 0,
+          errors: [`Worksheet '${section.sheetName}' not found`],
+          processedSections: sectionIndex + 1,
+          totalSections
+        });
+        continue;
+      }
+
+      const sectionData = await this.processSection(worksheet, section, sectionIndex, totalSections, onProgress);
+      results.push(sectionData);
+    }
+
+    return results;
+  }
+
+  private async processSection(
+    worksheet: ExcelJS.Worksheet,
+    section: any,
+    sectionIndex: number,
+    totalSections: number,
+    onProgress: (progress: ImportProgress) => void
+  ): Promise<ReadSectionData> {
+    const startingRow = section.startingRow || 1;
+    const endingRow = section.endingRow || worksheet.rowCount;
+    
+    // Get headers from the first row (assuming row 1 has headers)
+    const headers: string[] = [];
+    const headerRow = worksheet.getRow(1);
+    headerRow.eachCell((cell) => {
+      headers.push(cell.text || '');
+    });
+
+    // Process data rows
+    const data: Record<string, any>[] = [];
+    const totalRows = endingRow - startingRow + 1;
+    
+    for (let rowNum = startingRow; rowNum <= endingRow; rowNum++) {
+      const row = worksheet.getRow(rowNum);
+      if (!row.hasValues) continue;
+
+      const rowData: Record<string, any> = {};
+      
+      // Map cell values based on field configuration
+      for (const field of section.fields || []) {
+        const cellAddress = this.parseCellAddress(field.cellPosition);
+        const cell = row.getCell(cellAddress.col);
+        
+        if (cell && cell.value !== null && cell.value !== undefined) {
+          rowData[field.importTableColumnName] = this.convertCellValue(
+            cell.value,
+            field.parsedType || FieldTypeEnum.String
+          );
+        }
+      }
+      
+      data.push(rowData);
+      
+      // Update progress every 100 rows
+      if (rowNum % 100 === 0 || rowNum === endingRow) {
+        onProgress({
+          importId: 0,
+          status: 'processing',
+          currentSection: section.name,
+          currentRow: rowNum - startingRow + 1,
+          totalRows,
+          errors: [],
+          processedSections: sectionIndex,
+          totalSections
+        });
+      }
+    }
+
+    return {
+      id: section.id || 0,
+      name: section.name || '',
+      tableName: section.tableName || '',
+      sheet: section.sheetName || '',
+      type: section.type || '',
+      startingRow,
+      endingRow,
+      parsedType: this.mapSectionType(section.type),
+      fields: this.mapFields(section.fields || []),
+      data
+    };
+  }
+
+  private parseCellAddress(cellPosition: string): { row: number; col: number } {
+    const match = cellPosition.match(/([A-Z]+)(\d+)/);
+    if (!match) return { row: 1, col: 1 };
+    
+    const col = match[1].charCodeAt(0) - 'A'.charCodeAt(0) + 1;
+    const row = parseInt(match[2]);
+    
+    return { row, col };
+  }
+
+  private mapSectionType(type: string): SectionTypeEnum {
+    switch (type?.toLowerCase()) {
+      case 'grid':
+        return SectionTypeEnum.Grid;
+      case 'properties':
+        return SectionTypeEnum.Properties;
+      default:
+        return SectionTypeEnum.Unknown;
+    }
+  }
+
+  private mapFields(fields: any[]): LayoutSectionField[] {
+    return fields.map((field, index) => ({
+      id: field.id || index,
+      cellPosition: field.cellPosition || '',
+      name: field.name || '',
+      dataType: field.dataType || 'string',
+      dataTypeFormat: field.dataTypeFormat,
+      importTableColumnName: field.importTableColumnName || field.name || `column_${index}`,
+      importColumnOrderNumber: field.importColumnOrderNumber || index,
+      parsedType: this.mapFieldType(field.dataType)
+    }));
+  }
+
+  private mapFieldType(dataType: string): FieldTypeEnum {
+    const type = dataType?.toLowerCase();
+    
+    switch (type) {
+      case 'time':
+        return FieldTypeEnum.Time;
+      case 'decimal':
+      case 'number':
+      case 'float':
+        return FieldTypeEnum.Decimal;
+      case 'date':
+        return FieldTypeEnum.Date;
+      case 'int':
+      case 'integer':
+      case 'numeric':
+        return FieldTypeEnum.Numeric;
+      default:
+        return FieldTypeEnum.String;
+    }
+  }
+
+  private convertCellValue(value: any, fieldType: FieldTypeEnum): any {
+    if (value === null || value === undefined) return null;
+    
+    switch (fieldType) {
+      case FieldTypeEnum.Time:
+        if (typeof value === 'number') {
+          // Excel time is fraction of a day
+          return value * 24 * 60 * 60 * 1000; // Convert to milliseconds
+        }
+        return value;
+      
+      case FieldTypeEnum.Decimal:
+        return parseFloat(value.toString()) || 0;
+      
+      case FieldTypeEnum.Date:
+        if (typeof value === 'number') {
+          // Excel date is days since 1900-01-01
+          const excelEpoch = new Date(1900, 0, 1);
+          return new Date(excelEpoch.getTime() + (value - 1) * 24 * 60 * 60 * 1000);
+        }
+        return new Date(value);
+      
+      case FieldTypeEnum.Numeric:
+        return parseInt(value.toString()) || 0;
+      
+      case FieldTypeEnum.String:
+      default:
+        return value.toString();
+    }
+  }
+}

app/lib/excel-import/import-processor.ts

@@ -0,0 +1,183 @@
+import { PrismaClient } from '@prisma/client';
+import { ExcelReaderService } from './excel-reader';
+import { BulkInserter } from './bulk-inserter';
+import { ImportProgressServer } from './websocket-server';
+import { ImportProgress, ImportResult, ProcessedSection } from './types';
+
+export class ImportProcessor {
+  private prisma: PrismaClient;
+  private reader: ExcelReaderService;
+  private inserter: BulkInserter;
+  private progressServer: ImportProgressServer;
+
+  constructor() {
+    this.prisma = new PrismaClient();
+    this.reader = new ExcelReaderService();
+    this.inserter = new BulkInserter();
+    this.progressServer = new ImportProgressServer();
+  }
+
+  async processImport(importId: number): Promise<ImportResult> {
+    try {
+      // Get import record with layout configuration
+      const importRecord = await this.prisma.import.findUnique({
+        where: { id: importId },
+        include: { 
+          layout: { 
+            include: { 
+              sections: { 
+                include: { fields: true } 
+              } 
+            } 
+          },
+          file: true
+        }
+      });
+
+      if (!importRecord || !importRecord.file) {
+        throw new Error('Import not found or no file attached');
+      }
+
+      // Update import status to processing
+      await this.prisma.import.update({
+        where: { id: importId },
+        data: { status: 'PROCESSING' }
+      });
+
+      // Initialize progress tracking
+      const progress: ImportProgress = {
+        importId,
+        status: 'processing',
+        currentSection: '',
+        currentRow: 0,
+        totalRows: 0,
+        errors: [],
+        processedSections: 0,
+        totalSections: importRecord.layout?.sections?.length || 0
+      };
+
+      // Read Excel file
+      const sections = await this.reader.readExcelFile(
+        importRecord.file.data,
+        importRecord.layout,
+        (sectionProgress) => {
+          this.progressServer.broadcastProgress(importId, sectionProgress);
+        }
+      );
+
+      // Process each section
+      const processedSections: ProcessedSection[] = [];
+      let totalInserted = 0;
+
+      for (let i = 0; i < sections.length; i++) {
+        const section = sections[i];
+        
+        progress.currentSection = section.name;
+        progress.processedSections = i + 1;
+        this.progressServer.broadcastProgress(importId, progress);
+
+        try {
+          const insertedRows = await this.inserter.insertSectionData(
+            section,
+            importId,
+            (rows) => {
+              progress.currentRow = rows;
+              this.progressServer.broadcastProgress(importId, progress);
+            }
+          );
+
+          processedSections.push({
+            sectionData: section,
+            insertedRows
+          });
+
+          totalInserted += insertedRows;
+
+        } catch (error) {
+          const errorMessage = `Error processing section ${section.name}: ${error instanceof Error ? error.message : 'Unknown error'}`;
+          progress.errors.push(errorMessage);
+          this.progressServer.broadcastProgress(importId, progress);
+        }
+      }
+
+      // Update import status to completed
+      await this.prisma.import.update({
+        where: { id: importId },
+        data: { 
+          status: 'COMPLETED',
+          processedAt: new Date()
+        }
+      });
+
+      progress.status = 'completed';
+      this.progressServer.broadcastProgress(importId, progress);
+
+      return {
+        success: true,
+        totalInserted,
+        sections: processedSections
+      };
+
+    } catch (error) {
+      // Update import status to failed
+      await this.prisma.import.update({
+        where: { id: importId },
+        data: { 
+          status: 'FAILED',
+          error: error instanceof Error ? error.message : 'Unknown error',
+          processedAt: new Date()
+        }
+      });
+
+      const progress: ImportProgress = {
+        importId,
+        status: 'failed',
+        currentSection: '',
+        currentRow: 0,
+        totalRows: 0,
+        errors: [error instanceof Error ? error.message : 'Unknown error'],
+        processedSections: 0,
+        totalSections: 0
+      };
+
+      this.progressServer.broadcastProgress(importId, progress);
+
+      return {
+        success: false,
+        totalInserted: 0,
+        sections: [],
+        errors: [error instanceof Error ? error.message : 'Unknown error']
+      };
+    }
+  }
+
+  async validateImport(importId: number): Promise<{ valid: boolean; errors: string[] }> {
+    const errors: string[] = [];
+
+    try {
+      const importRecord = await this.prisma.import.findUnique({
+        where: { id: importId },
+        include: { file: true, layout: true }
+      });
+
+      if (!importRecord) {
+        errors.push('Import record not found');
+        return { valid: false, errors };
+      }
+
+      if (!importRecord.file) {
+        errors.push('No file attached to import');
+      }
+
+      if (!importRecord.layout) {
+        errors.push('No layout configuration found');
+      }
+
+      return { valid: errors.length === 0, errors };
+
+    } catch (error) {
+      errors.push(`Validation error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+      return { valid: false, errors };
+    }
+  }
+}

app/lib/excel-import/types.ts

@@ -0,0 +1,68 @@
+// TypeScript interfaces matching C# models from Excelerator codebase
+
+export interface ReadSectionData {
+  id: number;
+  name: string;
+  tableName: string;
+  sheet: string;
+  type: string;
+  startingRow?: number;
+  endingRow?: number;
+  parsedType: SectionTypeEnum;
+  fields: LayoutSectionField[];
+  data: Record<string, any>[];
+}
+
+export interface LayoutSectionField {
+  id: number;
+  cellPosition: string;
+  name: string;
+  dataType: string;
+  dataTypeFormat?: string;
+  importTableColumnName: string;
+  importColumnOrderNumber: number;
+  parsedType: FieldTypeEnum;
+}
+
+export enum SectionTypeEnum {
+  Grid = 'Grid',
+  Properties = 'Properties',
+  Unknown = 'Unknown'
+}
+
+export enum FieldTypeEnum {
+  Time = 'Time',
+  Decimal = 'Decimal',
+  Date = 'Date',
+  Numeric = 'Numeric',
+  String = 'String'
+}
+
+export interface ImportProgress {
+  importId: number;
+  status: 'pending' | 'processing' | 'completed' | 'failed';
+  currentSection: string;
+  currentRow: number;
+  totalRows: number;
+  errors: string[];
+  processedSections: number;
+  totalSections: number;
+}
+
+export interface ExcelImportConfig {
+  fileBuffer: Buffer;
+  layoutConfiguration: any;
+  importId: number;
+}
+
+export interface ProcessedSection {
+  sectionData: ReadSectionData;
+  insertedRows: number;
+}
+
+export interface ImportResult {
+  success: boolean;
+  totalInserted: number;
+  sections: ProcessedSection[];
+  errors?: string[];
+}

app/lib/excel-import/websocket-server.ts

@@ -0,0 +1,127 @@
+import { WebSocketServer, WebSocket } from 'ws';
+import { Server } from 'http';
+import { ImportProgress } from './types';
+
+export class ImportProgressServer {
+  private wss: WebSocketServer | null = null;
+  private clients: Set<WebSocket> = new Set();
+
+  constructor() {
+    // Initialize WebSocket server
+    this.setupWebSocketServer();
+  }
+
+  private setupWebSocketServer() {
+    // This will be initialized when the HTTP server is available
+    // For now, we'll create a standalone WebSocket server
+    this.wss = new WebSocketServer({ port: 8080 });
+    
+    this.wss.on('connection', (ws: WebSocket) => {
+      console.log('New WebSocket connection established');
+      this.clients.add(ws);
+
+      ws.on('close', () => {
+        console.log('WebSocket connection closed');
+        this.clients.delete(ws);
+      });
+
+      ws.on('error', (error) => {
+        console.error('WebSocket error:', error);
+        this.clients.delete(ws);
+      });
+    });
+
+    console.log('WebSocket server started on port 8080');
+  }
+
+  // Alternative method to attach to existing HTTP server
+  attachToServer(server: Server) {
+    if (this.wss) {
+      this.wss.close();
+    }
+
+    this.wss = new WebSocketServer({ 
+      server, 
+      path: '/api/imports/progress' 
+    });
+
+    this.wss.on('connection', (ws: WebSocket, request) => {
+      console.log('New WebSocket connection on import progress endpoint');
+      this.clients.add(ws);
+
+      // Handle specific import ID subscriptions
+      const url = new URL(request.url || '', `http://${request.headers.host}`);
+      const importId = url.searchParams.get('importId');
+      
+      if (importId) {
+        ws.send(JSON.stringify({
+          type: 'connected',
+          importId: parseInt(importId),
+          message: 'Connected to import progress updates'
+        }));
+      }
+
+      ws.on('close', () => {
+        console.log('WebSocket connection closed');
+        this.clients.delete(ws);
+      });
+
+      ws.on('error', (error) => {
+        console.error('WebSocket error:', error);
+        this.clients.delete(ws);
+      });
+    });
+  }
+
+  broadcastProgress(importId: number, progress: ImportProgress) {
+    const message = JSON.stringify({
+      type: 'progress',
+      importId,
+      progress
+    });
+
+    this.clients.forEach(client => {
+      if (client.readyState === WebSocket.OPEN) {
+        client.send(message);
+      }
+    });
+  }
+
+  broadcastError(importId: number, error: string) {
+    const message = JSON.stringify({
+      type: 'error',
+      importId,
+      error
+    });
+
+    this.clients.forEach(client => {
+      if (client.readyState === WebSocket.OPEN) {
+        client.send(message);
+      }
+    });
+  }
+
+  broadcastComplete(importId: number, result: any) {
+    const message = JSON.stringify({
+      type: 'complete',
+      importId,
+      result
+    });
+
+    this.clients.forEach(client => {
+      if (client.readyState === WebSocket.OPEN) {
+        client.send(message);
+      }
+    });
+  }
+
+  close() {
+    if (this.wss) {
+      this.wss.close();
+      this.clients.clear();
+    }
+  }
+}
+
+// Export a singleton instance
+export const progressServer = new ImportProgressServer();

package.json

@@ -31,6 +31,7 @@
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
     "date-fns": "^4.1.0",
+    "exceljs": "^4.4.0",
     "lucide-react": "^0.525.0",
     "next": "^15.4.1",
     "pg": "^8.16.3",
@@ -39,6 +40,7 @@
     "react-hook-form": "^7.60.0",
     "tailwind-merge": "^3.3.1",
     "tailwindcss-animate": "^1.0.7",
+    "ws": "^8.18.3",
     "zod": "^4.0.5"
   },
   "devDependencies": {
@@ -46,6 +48,7 @@
     "@types/node": "^20",
     "@types/react": "^19",
     "@types/react-dom": "^19",
+    "@types/ws": "^8.18.1",
     "eslint": "^9",
     "eslint-config-next": "15.1.6",
     "postcss": "^8",