|
@@ -1,49 +1,19 @@
|
|
|
-/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
|
|
|
-/* eslint-disable @typescript-eslint/no-unused-vars */
|
|
|
|
|
import * as XLSX from 'xlsx';
|
|
import * as XLSX from 'xlsx';
|
|
|
import { ReadSectionData, LayoutSectionField, SectionTypeEnum, FieldTypeEnum, ImportProgress } from './types';
|
|
import { ReadSectionData, LayoutSectionField, SectionTypeEnum, FieldTypeEnum, ImportProgress } from './types';
|
|
|
|
|
|
|
|
-// Simple logger utility for debugging
|
|
|
|
|
-const logger = {
|
|
|
|
|
- debug: (message: string, ...args: any[]) => {
|
|
|
|
|
- //console.debug(`[ExcelReaderService] ${new Date().toISOString()} - ${message}`, ...args);
|
|
|
|
|
- },
|
|
|
|
|
- info: (message: string, ...args: any[]) => {
|
|
|
|
|
- //console.info(`[ExcelReaderService] ${new Date().toISOString()} - ${message}`, ...args);
|
|
|
|
|
- },
|
|
|
|
|
- warn: (message: string, ...args: any[]) => {
|
|
|
|
|
- //console.warn(`[ExcelReaderService] ${new Date().toISOString()} - ${message}`, ...args);
|
|
|
|
|
- },
|
|
|
|
|
- error: (message: string, ...args: any[]) => {
|
|
|
|
|
- //console.error(`[ExcelReaderService] ${new Date().toISOString()} - ${message}`, ...args);
|
|
|
|
|
- }
|
|
|
|
|
-};
|
|
|
|
|
-
|
|
|
|
|
export class ExcelReaderService {
|
|
export class ExcelReaderService {
|
|
|
async readExcelFile(
|
|
async readExcelFile(
|
|
|
fileData: Buffer | Uint8Array,
|
|
fileData: Buffer | Uint8Array,
|
|
|
layoutConfig: any,
|
|
layoutConfig: any,
|
|
|
onProgress: (progress: ImportProgress) => void
|
|
onProgress: (progress: ImportProgress) => void
|
|
|
): Promise<ReadSectionData[]> {
|
|
): Promise<ReadSectionData[]> {
|
|
|
- logger.info('Starting Excel file import', {
|
|
|
|
|
- fileSize: fileData.length,
|
|
|
|
|
- layoutConfigSections: layoutConfig.sections?.length || 0
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
- const startTime = Date.now();
|
|
|
|
|
|
|
|
|
|
try {
|
|
try {
|
|
|
- logger.debug('Loading Excel workbook from buffer...');
|
|
|
|
|
const workbook = XLSX.read(fileData, { type: 'buffer' });
|
|
const workbook = XLSX.read(fileData, { type: 'buffer' });
|
|
|
- logger.info('Excel workbook loaded successfully', {
|
|
|
|
|
- worksheets: workbook.SheetNames.map(name => ({ name, rowCount: XLSX.utils.sheet_to_json(workbook.Sheets[name]).length }))
|
|
|
|
|
- });
|
|
|
|
|
|
|
|
|
|
const results: ReadSectionData[] = [];
|
|
const results: ReadSectionData[] = [];
|
|
|
const totalSections = layoutConfig.sections?.length || 0;
|
|
const totalSections = layoutConfig.sections?.length || 0;
|
|
|
|
|
|
|
|
- logger.info('Processing Excel import', { totalSections });
|
|
|
|
|
-
|
|
|
|
|
// Initialize progress
|
|
// Initialize progress
|
|
|
onProgress({
|
|
onProgress({
|
|
|
importId: 0, // Will be set by caller
|
|
importId: 0, // Will be set by caller
|
|
@@ -58,18 +28,11 @@ export class ExcelReaderService {
|
|
|
|
|
|
|
|
for (let sectionIndex = 0; sectionIndex < totalSections; sectionIndex++) {
|
|
for (let sectionIndex = 0; sectionIndex < totalSections; sectionIndex++) {
|
|
|
const section = layoutConfig.sections[sectionIndex];
|
|
const section = layoutConfig.sections[sectionIndex];
|
|
|
- logger.debug(`Processing section ${sectionIndex + 1}/${totalSections}`, {
|
|
|
|
|
- sectionName: section.name,
|
|
|
|
|
- sheetName: section.sheetName,
|
|
|
|
|
- startingRow: section.startingRow,
|
|
|
|
|
- endingRow: section.endingRow
|
|
|
|
|
- });
|
|
|
|
|
|
|
|
|
|
const worksheet = workbook.Sheets[section.sheetName];
|
|
const worksheet = workbook.Sheets[section.sheetName];
|
|
|
|
|
|
|
|
if (!worksheet) {
|
|
if (!worksheet) {
|
|
|
const error = `Worksheet '${section.sheetName}' not found`;
|
|
const error = `Worksheet '${section.sheetName}' not found`;
|
|
|
- logger.warn(error, { availableWorksheets: workbook.SheetNames });
|
|
|
|
|
|
|
|
|
|
onProgress({
|
|
onProgress({
|
|
|
importId: 0,
|
|
importId: 0,
|
|
@@ -86,26 +49,10 @@ export class ExcelReaderService {
|
|
|
|
|
|
|
|
const sectionData = await this.processSection(worksheet, section, sectionIndex, totalSections, onProgress);
|
|
const sectionData = await this.processSection(worksheet, section, sectionIndex, totalSections, onProgress);
|
|
|
results.push(sectionData);
|
|
results.push(sectionData);
|
|
|
-
|
|
|
|
|
- logger.info(`Section ${section.name} processed successfully`, {
|
|
|
|
|
- rowsProcessed: sectionData.data.length,
|
|
|
|
|
- fields: sectionData.fields.length
|
|
|
|
|
- });
|
|
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- const totalTime = Date.now() - startTime;
|
|
|
|
|
- logger.info('Excel file import completed', {
|
|
|
|
|
- totalSections: results.length,
|
|
|
|
|
- totalRows: results.reduce((sum, section) => sum + section.data.length, 0),
|
|
|
|
|
- totalTimeMs: totalTime
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
return results;
|
|
return results;
|
|
|
} catch (error) {
|
|
} catch (error) {
|
|
|
- logger.error('Error reading Excel file', {
|
|
|
|
|
- error: error instanceof Error ? error.message : String(error),
|
|
|
|
|
- stack: error instanceof Error ? error.stack : undefined
|
|
|
|
|
- });
|
|
|
|
|
throw error;
|
|
throw error;
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
@@ -117,24 +64,9 @@ export class ExcelReaderService {
|
|
|
totalSections: number,
|
|
totalSections: number,
|
|
|
onProgress: (progress: ImportProgress) => void
|
|
onProgress: (progress: ImportProgress) => void
|
|
|
): Promise<ReadSectionData> {
|
|
): Promise<ReadSectionData> {
|
|
|
- const sectionStartTime = Date.now();
|
|
|
|
|
- logger.info(`Starting section processing`, {
|
|
|
|
|
- sectionName: section.name,
|
|
|
|
|
- sheetName: section.sheetName,
|
|
|
|
|
- sectionIndex: sectionIndex + 1,
|
|
|
|
|
- totalSections
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
const startingRow = section.startingRow || 2; // Default to 2 to skip header
|
|
const startingRow = section.startingRow || 2; // Default to 2 to skip header
|
|
|
const endingRow = section.endingRow || Infinity;
|
|
const endingRow = section.endingRow || Infinity;
|
|
|
|
|
|
|
|
- logger.debug('Section configuration', {
|
|
|
|
|
- sectionName: section.name,
|
|
|
|
|
- startingRow,
|
|
|
|
|
- endingRow,
|
|
|
|
|
- fieldsCount: section.fields?.length || 0
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
// Convert worksheet to JSON array
|
|
// Convert worksheet to JSON array
|
|
|
const worksheetData = XLSX.utils.sheet_to_json(worksheet, { header: 1 }) as any[][];
|
|
const worksheetData = XLSX.utils.sheet_to_json(worksheet, { header: 1 }) as any[][];
|
|
|
|
|
|
|
@@ -142,20 +74,14 @@ export class ExcelReaderService {
|
|
|
const data: Record<string, any>[] = [];
|
|
const data: Record<string, any>[] = [];
|
|
|
const totalRows = Math.min(endingRow, worksheetData.length) - startingRow + 1;
|
|
const totalRows = Math.min(endingRow, worksheetData.length) - startingRow + 1;
|
|
|
|
|
|
|
|
- let processedRows = 0;
|
|
|
|
|
- let skippedRows = 0;
|
|
|
|
|
-
|
|
|
|
|
for (let rowNum = startingRow; rowNum <= Math.min(endingRow, worksheetData.length); rowNum++) {
|
|
for (let rowNum = startingRow; rowNum <= Math.min(endingRow, worksheetData.length); rowNum++) {
|
|
|
const row = worksheetData[rowNum - 1]; // Convert to 0-based index
|
|
const row = worksheetData[rowNum - 1]; // Convert to 0-based index
|
|
|
|
|
|
|
|
if (!row || row.every(cell => cell === null || cell === undefined || cell === '')) {
|
|
if (!row || row.every(cell => cell === null || cell === undefined || cell === '')) {
|
|
|
- skippedRows++;
|
|
|
|
|
- logger.debug(`Skipping empty row ${rowNum}`);
|
|
|
|
|
continue;
|
|
continue;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
const rowData: Record<string, any> = {};
|
|
const rowData: Record<string, any> = {};
|
|
|
- let fieldsProcessed = 0;
|
|
|
|
|
|
|
|
|
|
// Map cell values based on field configuration
|
|
// Map cell values based on field configuration
|
|
|
for (const field of section.fields || []) {
|
|
for (const field of section.fields || []) {
|
|
@@ -163,56 +89,24 @@ export class ExcelReaderService {
|
|
|
const cellAddress = this.parseCellAddress(field.cellPosition);
|
|
const cellAddress = this.parseCellAddress(field.cellPosition);
|
|
|
const cellValue = row[cellAddress.col - 1]; // Convert to 0-based index
|
|
const cellValue = row[cellAddress.col - 1]; // Convert to 0-based index
|
|
|
|
|
|
|
|
- logger.debug(`Processing field`, {
|
|
|
|
|
- fieldName: field.name,
|
|
|
|
|
- cellPosition: field.cellPosition,
|
|
|
|
|
- cellAddress,
|
|
|
|
|
- rawValue: cellValue,
|
|
|
|
|
- rowNum
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
if (cellValue !== null && cellValue !== undefined && cellValue !== '') {
|
|
if (cellValue !== null && cellValue !== undefined && cellValue !== '') {
|
|
|
const value = this.convertCellValue(
|
|
const value = this.convertCellValue(
|
|
|
cellValue,
|
|
cellValue,
|
|
|
field.parsedType || FieldTypeEnum.String
|
|
field.parsedType || FieldTypeEnum.String
|
|
|
);
|
|
);
|
|
|
|
|
|
|
|
- logger.debug(`Value converted`, {
|
|
|
|
|
- fieldName: field.name,
|
|
|
|
|
- originalValue: cellValue,
|
|
|
|
|
- convertedValue: value,
|
|
|
|
|
- fieldType: field.parsedType || FieldTypeEnum.String
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
// Map to the correct column name for Prisma model
|
|
// Map to the correct column name for Prisma model
|
|
|
const columnName = field.importTableColumnName;
|
|
const columnName = field.importTableColumnName;
|
|
|
rowData[columnName] = value;
|
|
rowData[columnName] = value;
|
|
|
- fieldsProcessed++;
|
|
|
|
|
}
|
|
}
|
|
|
- } catch (error) {
|
|
|
|
|
- logger.error(`Error processing field ${field.name} at row ${rowNum}`, {
|
|
|
|
|
- error: error instanceof Error ? error.message : String(error),
|
|
|
|
|
- field,
|
|
|
|
|
- rowNum
|
|
|
|
|
- });
|
|
|
|
|
|
|
+ } catch {
|
|
|
|
|
+ // Error processing field, skip this field
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// Only add non-empty rows
|
|
// Only add non-empty rows
|
|
|
if (Object.keys(rowData).length > 0) {
|
|
if (Object.keys(rowData).length > 0) {
|
|
|
data.push(rowData);
|
|
data.push(rowData);
|
|
|
- processedRows++;
|
|
|
|
|
-
|
|
|
|
|
- if (processedRows <= 5 || processedRows % 100 === 0) {
|
|
|
|
|
- logger.debug(`Row processed`, {
|
|
|
|
|
- rowNum,
|
|
|
|
|
- fieldsProcessed,
|
|
|
|
|
- rowDataKeys: Object.keys(rowData),
|
|
|
|
|
- dataLength: data.length
|
|
|
|
|
- });
|
|
|
|
|
- }
|
|
|
|
|
- } else {
|
|
|
|
|
- logger.debug(`Skipping row with no valid data`, { rowNum });
|
|
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// Update progress every 100 rows
|
|
// Update progress every 100 rows
|
|
@@ -230,16 +124,6 @@ export class ExcelReaderService {
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- const sectionTime = Date.now() - sectionStartTime;
|
|
|
|
|
- logger.info(`Section processing completed`, {
|
|
|
|
|
- sectionName: section.name,
|
|
|
|
|
- processedRows,
|
|
|
|
|
- skippedRows,
|
|
|
|
|
- totalRows,
|
|
|
|
|
- dataRows: data.length,
|
|
|
|
|
- processingTimeMs: sectionTime
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
const result = {
|
|
const result = {
|
|
|
id: section.id || 0,
|
|
id: section.id || 0,
|
|
|
name: section.name || '',
|
|
name: section.name || '',
|
|
@@ -253,44 +137,22 @@ export class ExcelReaderService {
|
|
|
data
|
|
data
|
|
|
};
|
|
};
|
|
|
|
|
|
|
|
- logger.debug('Section result', {
|
|
|
|
|
- sectionName: section.name,
|
|
|
|
|
- resultSummary: {
|
|
|
|
|
- id: result.id,
|
|
|
|
|
- name: result.name,
|
|
|
|
|
- tableName: result.tableName,
|
|
|
|
|
- dataRows: result.data.length,
|
|
|
|
|
- fields: result.fields.length
|
|
|
|
|
- }
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
return result;
|
|
return result;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
private parseCellAddress(cellPosition: string): { row: number; col: number } {
|
|
private parseCellAddress(cellPosition: string): { row: number; col: number } {
|
|
|
- logger.debug(`Parsing cell address: ${cellPosition}`);
|
|
|
|
|
-
|
|
|
|
|
const match = cellPosition.match(/([A-Z]+)(\d+)/);
|
|
const match = cellPosition.match(/([A-Z]+)(\d+)/);
|
|
|
if (!match) {
|
|
if (!match) {
|
|
|
- logger.warn(`Invalid cell position format: ${cellPosition}, using default 1,1`);
|
|
|
|
|
return { row: 1, col: 1 };
|
|
return { row: 1, col: 1 };
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
const col = match[1].charCodeAt(0) - 'A'.charCodeAt(0) + 1;
|
|
const col = match[1].charCodeAt(0) - 'A'.charCodeAt(0) + 1;
|
|
|
const row = parseInt(match[2]);
|
|
const row = parseInt(match[2]);
|
|
|
|
|
|
|
|
- logger.debug(`Parsed cell address`, {
|
|
|
|
|
- original: cellPosition,
|
|
|
|
|
- row,
|
|
|
|
|
- col
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
return { row, col };
|
|
return { row, col };
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
private mapSectionType(type: string): SectionTypeEnum {
|
|
private mapSectionType(type: string): SectionTypeEnum {
|
|
|
- logger.debug(`Mapping section type: ${type}`);
|
|
|
|
|
-
|
|
|
|
|
const mappedType = (() => {
|
|
const mappedType = (() => {
|
|
|
switch (type?.toLowerCase()) {
|
|
switch (type?.toLowerCase()) {
|
|
|
case 'grid':
|
|
case 'grid':
|
|
@@ -302,17 +164,10 @@ export class ExcelReaderService {
|
|
|
}
|
|
}
|
|
|
})();
|
|
})();
|
|
|
|
|
|
|
|
- logger.debug(`Section type mapped`, {
|
|
|
|
|
- originalType: type,
|
|
|
|
|
- mappedType: SectionTypeEnum[mappedType]
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
return mappedType;
|
|
return mappedType;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
private mapFields(fields: any[]): LayoutSectionField[] {
|
|
private mapFields(fields: any[]): LayoutSectionField[] {
|
|
|
- logger.debug(`Mapping ${fields.length} fields`);
|
|
|
|
|
-
|
|
|
|
|
const mappedFields = fields.map((field, index) => {
|
|
const mappedFields = fields.map((field, index) => {
|
|
|
const mappedField = {
|
|
const mappedField = {
|
|
|
id: field.id || index,
|
|
id: field.id || index,
|
|
@@ -325,14 +180,6 @@ export class ExcelReaderService {
|
|
|
parsedType: this.mapFieldType(field.dataType)
|
|
parsedType: this.mapFieldType(field.dataType)
|
|
|
};
|
|
};
|
|
|
|
|
|
|
|
- logger.debug(`Field mapped`, {
|
|
|
|
|
- index,
|
|
|
|
|
- originalName: field.name,
|
|
|
|
|
- mappedName: mappedField.name,
|
|
|
|
|
- cellPosition: mappedField.cellPosition,
|
|
|
|
|
- parsedType: FieldTypeEnum[mappedField.parsedType]
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
return mappedField;
|
|
return mappedField;
|
|
|
});
|
|
});
|
|
|
|
|
|
|
@@ -361,40 +208,26 @@ export class ExcelReaderService {
|
|
|
}
|
|
}
|
|
|
})();
|
|
})();
|
|
|
|
|
|
|
|
- logger.debug(`Field type mapped`, {
|
|
|
|
|
- originalDataType: dataType,
|
|
|
|
|
- mappedType: FieldTypeEnum[mappedType]
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
return mappedType;
|
|
return mappedType;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
private convertCellValue(value: any, fieldType: FieldTypeEnum): any {
|
|
private convertCellValue(value: any, fieldType: FieldTypeEnum): any {
|
|
|
if (value === null || value === undefined) {
|
|
if (value === null || value === undefined) {
|
|
|
- logger.debug(`Converting null/undefined value to null`, { fieldType: FieldTypeEnum[fieldType] });
|
|
|
|
|
return null;
|
|
return null;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- logger.debug(`Converting cell value`, {
|
|
|
|
|
- originalValue: value,
|
|
|
|
|
- originalType: typeof value,
|
|
|
|
|
- targetFieldType: FieldTypeEnum[fieldType]
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
const convertedValue = (() => {
|
|
const convertedValue = (() => {
|
|
|
switch (fieldType) {
|
|
switch (fieldType) {
|
|
|
case FieldTypeEnum.Time:
|
|
case FieldTypeEnum.Time:
|
|
|
if (typeof value === 'number') {
|
|
if (typeof value === 'number') {
|
|
|
// Excel time is fraction of a day
|
|
// Excel time is fraction of a day
|
|
|
const result = value * 24 * 60 * 60 * 1000; // Convert to milliseconds
|
|
const result = value * 24 * 60 * 60 * 1000; // Convert to milliseconds
|
|
|
- logger.debug(`Time conversion`, { original: value, converted: result });
|
|
|
|
|
return result;
|
|
return result;
|
|
|
}
|
|
}
|
|
|
return value;
|
|
return value;
|
|
|
|
|
|
|
|
case FieldTypeEnum.Decimal:
|
|
case FieldTypeEnum.Decimal:
|
|
|
const decimalResult = parseFloat(value.toString()) || 0;
|
|
const decimalResult = parseFloat(value.toString()) || 0;
|
|
|
- logger.debug(`Decimal conversion`, { original: value, converted: decimalResult });
|
|
|
|
|
return decimalResult;
|
|
return decimalResult;
|
|
|
|
|
|
|
|
case FieldTypeEnum.Date:
|
|
case FieldTypeEnum.Date:
|
|
@@ -402,22 +235,18 @@ export class ExcelReaderService {
|
|
|
// Excel date is days since 1900-01-01
|
|
// Excel date is days since 1900-01-01
|
|
|
const excelEpoch = new Date(1900, 0, 1);
|
|
const excelEpoch = new Date(1900, 0, 1);
|
|
|
const dateResult = new Date(excelEpoch.getTime() + (value - 1) * 24 * 60 * 60 * 1000);
|
|
const dateResult = new Date(excelEpoch.getTime() + (value - 1) * 24 * 60 * 60 * 1000);
|
|
|
- logger.debug(`Date conversion`, { original: value, converted: dateResult });
|
|
|
|
|
return dateResult;
|
|
return dateResult;
|
|
|
}
|
|
}
|
|
|
const dateResult = new Date(value);
|
|
const dateResult = new Date(value);
|
|
|
- logger.debug(`Date conversion from string`, { original: value, converted: dateResult });
|
|
|
|
|
return dateResult;
|
|
return dateResult;
|
|
|
|
|
|
|
|
case FieldTypeEnum.Numeric:
|
|
case FieldTypeEnum.Numeric:
|
|
|
const numericResult = parseInt(value.toString()) || 0;
|
|
const numericResult = parseInt(value.toString()) || 0;
|
|
|
- logger.debug(`Numeric conversion`, { original: value, converted: numericResult });
|
|
|
|
|
return numericResult;
|
|
return numericResult;
|
|
|
|
|
|
|
|
case FieldTypeEnum.String:
|
|
case FieldTypeEnum.String:
|
|
|
default:
|
|
default:
|
|
|
const stringResult = value.toString();
|
|
const stringResult = value.toString();
|
|
|
- logger.debug(`String conversion`, { original: value, converted: stringResult });
|
|
|
|
|
return stringResult;
|
|
return stringResult;
|
|
|
}
|
|
}
|
|
|
})();
|
|
})();
|