excel-reader.ts 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319
  1. import * as XLSX from 'xlsx';
  2. import { ReadSectionData, LayoutSectionField, SectionTypeEnum, FieldTypeEnum, ImportProgress } from './types';
  3. export class ExcelReaderService {
  4. async readExcelFile(
  5. fileData: Buffer | Uint8Array,
  6. layoutConfig: any,
  7. onProgress: (progress: ImportProgress) => void
  8. ): Promise<ReadSectionData[]> {
  9. console.log('[EXCEL_READER] 📖 Starting readExcelFile');
  10. console.log('[EXCEL_READER] 📊 File data type:', typeof fileData, fileData instanceof Buffer, fileData instanceof Uint8Array);
  11. console.log('[EXCEL_READER] 📊 Layout config sections:', layoutConfig?.sections?.length);
  12. try {
  13. console.log('[EXCEL_READER] 🔄 Parsing workbook...');
  14. const workbook = XLSX.read(fileData, { type: 'buffer' });
  15. console.log('[EXCEL_READER] 📋 Workbook sheet names:', workbook.SheetNames);
  16. const results: ReadSectionData[] = [];
  17. const totalSections = layoutConfig.sections?.length || 0;
  18. // Initialize progress
  19. onProgress({
  20. importId: 0, // Will be set by caller
  21. status: 'processing',
  22. currentSection: '',
  23. currentRow: 0,
  24. totalRows: 0,
  25. errors: [],
  26. processedSections: 0,
  27. totalSections
  28. });
  29. for (let sectionIndex = 0; sectionIndex < totalSections; sectionIndex++) {
  30. const section = layoutConfig.sections[sectionIndex];
  31. console.log(`[EXCEL_READER] 📄 Processing section ${sectionIndex + 1}/${totalSections}:`, {
  32. name: section.name,
  33. sheetName: section.sheetName,
  34. tableName: section.tableName,
  35. fieldsCount: section.fields?.length
  36. });
  37. const worksheet = workbook.Sheets[section.sheetName];
  38. if (!worksheet) {
  39. console.error(`[EXCEL_READER] ❌ Worksheet '${section.sheetName}' not found! Available sheets:`, workbook.SheetNames);
  40. const error = `Worksheet '${section.sheetName}' not found`;
  41. onProgress({
  42. importId: 0,
  43. status: 'processing',
  44. currentSection: section.name,
  45. currentRow: 0,
  46. totalRows: 0,
  47. errors: [error],
  48. processedSections: sectionIndex + 1,
  49. totalSections
  50. });
  51. continue;
  52. }
  53. console.log(`[EXCEL_READER] ✅ Found worksheet '${section.sheetName}'`);
  54. const sectionData = await this.processSection(worksheet, section, sectionIndex, totalSections, onProgress);
  55. console.log(`[EXCEL_READER] ✅ Section '${section.name}' processed:`, { dataRows: sectionData.data?.length });
  56. results.push(sectionData);
  57. }
  58. console.log('[EXCEL_READER] ✅ All sections processed. Total rows across all sections:', results.reduce((sum, s) => sum + (s.data?.length || 0), 0));
  59. return results;
  60. } catch (error) {
  61. console.error('[EXCEL_READER] ❌ Error reading Excel file:', error);
  62. throw error;
  63. }
  64. }
  65. private async processSection(
  66. worksheet: XLSX.WorkSheet,
  67. section: any,
  68. sectionIndex: number,
  69. totalSections: number,
  70. onProgress: (progress: ImportProgress) => void
  71. ): Promise<ReadSectionData> {
  72. const startingRow = section.startingRow || 2; // Default to 2 to skip header
  73. const endingRow = section.endingRow || Infinity;
  74. // Convert worksheet to JSON array
  75. const worksheetData = XLSX.utils.sheet_to_json(worksheet, { header: 1 }) as any[][];
  76. console.log(`[EXCEL_READER] Worksheet data shape:`, { rows: worksheetData.length, cols: worksheetData[0]?.length });
  77. // Process data rows
  78. const data: Record<string, any>[] = [];
  79. const totalRows = Math.min(endingRow, worksheetData.length) - startingRow + 1;
  80. console.log(`[EXCEL_READER] Processing rows ${startingRow} to ${Math.min(endingRow, worksheetData.length)} (${totalRows} total rows)`);
  81. for (let rowNum = startingRow; rowNum <= Math.min(endingRow, worksheetData.length); rowNum++) {
  82. const row = worksheetData[rowNum - 1]; // Convert to 0-based index
  83. if (!row || row.every(cell => cell === null || cell === undefined || cell === '')) {
  84. continue;
  85. }
  86. const rowData: Record<string, any> = {};
  87. // Map cell values based on field configuration
  88. for (const field of section.fields || []) {
  89. try {
  90. const cellAddress = this.parseCellAddress(field.cellPosition, rowNum);
  91. const cellValue = row[cellAddress.col - 1]; // Convert to 0-based index
  92. if (cellValue !== null && cellValue !== undefined && cellValue !== '') {
  93. const value = this.convertCellValue(
  94. field.dataType,
  95. field.dataTypeFormat,
  96. cellValue,
  97. field.parsedType || FieldTypeEnum.String
  98. );
  99. // Map to the correct column name for Prisma model
  100. const columnName = field.importTableColumnName;
  101. rowData[columnName] = value;
  102. }
  103. } catch (error) {
  104. console.log(`[EXCEL_READER] Error processing field ${field.name} at row ${rowNum}`, {
  105. error: error instanceof Error ? error.message : String(error),
  106. field,
  107. rowNum
  108. });
  109. }
  110. }
  111. // Only add non-empty, non-header rows
  112. if (Object.keys(rowData).length > 0 && !this.isCintasHeaderRow(section.tableName, rowData)) {
  113. data.push(rowData);
  114. }
  115. // Update progress every 100 rows
  116. if (rowNum % 100 === 0 || rowNum === Math.min(endingRow, worksheetData.length)) {
  117. onProgress({
  118. importId: 0,
  119. status: 'processing',
  120. currentSection: section.name,
  121. currentRow: rowNum - startingRow + 1,
  122. totalRows,
  123. errors: [],
  124. processedSections: sectionIndex,
  125. totalSections
  126. });
  127. }
  128. }
  129. console.log(`[EXCEL_READER] Final section data:`, {
  130. name: section.name,
  131. totalRows: data.length,
  132. fieldsMapped: Object.keys(data[0] || {}).length
  133. });
  134. const result = {
  135. id: section.id || 0,
  136. name: section.name || '',
  137. tableName: section.tableName || '',
  138. sheet: section.sheetName || '',
  139. type: section.type || '',
  140. startingRow,
  141. endingRow,
  142. parsedType: this.mapSectionType(section.type),
  143. fields: this.mapFields(section.fields || []),
  144. data
  145. };
  146. return result;
  147. }
  148. private isCintasHeaderRow(tableName: string, rowData: Record<string, any>): boolean {
  149. if (tableName !== 'cintas_install_calendar') {
  150. return false;
  151. }
  152. const values = Object.values(rowData)
  153. .map((value) => (typeof value === 'string' ? value.trim().toLowerCase() : ''))
  154. .filter(Boolean);
  155. if (values.length === 0) {
  156. return false;
  157. }
  158. if (values.includes('date of install')) {
  159. return true;
  160. }
  161. const hasOpportunityHeader = values.includes('opportunity status');
  162. const hasTrrHeader = values.includes('trr');
  163. return hasOpportunityHeader && hasTrrHeader;
  164. }
  165. private parseCellAddress(cellPosition: string, rowNumber: number): { row: number; col: number } {
  166. let match = cellPosition.match(/([A-Z]+)(\d+)/);
  167. if (!match) {
  168. const appendedCellPosition = `${cellPosition}${rowNumber}`;
  169. match = appendedCellPosition.match(/([A-Z]+)(\d+)/);
  170. if (!match) {
  171. return { row: 1, col: 1 };
  172. }
  173. }
  174. const col = match[1].charCodeAt(0) - 'A'.charCodeAt(0) + 1;
  175. const row = parseInt(match[2]);
  176. return { row, col };
  177. }
  178. private mapSectionType(type: string): SectionTypeEnum {
  179. const mappedType = (() => {
  180. switch (type?.toLowerCase()) {
  181. case 'grid':
  182. return SectionTypeEnum.Grid;
  183. case 'properties':
  184. return SectionTypeEnum.Properties;
  185. default:
  186. return SectionTypeEnum.Unknown;
  187. }
  188. })();
  189. return mappedType;
  190. }
  191. private mapFields(fields: any[]): LayoutSectionField[] {
  192. const mappedFields = fields.map((field, index) => {
  193. const mappedField = {
  194. id: field.id || index,
  195. cellPosition: field.cellPosition || '',
  196. name: field.name || '',
  197. dataType: field.dataType || 'string',
  198. dataTypeFormat: field.dataTypeFormat,
  199. importTableColumnName: field.importTableColumnName || field.name || `column_${index}`,
  200. importColumnOrderNumber: field.importColumnOrderNumber || index,
  201. parsedType: this.mapFieldType(field.dataType)
  202. };
  203. return mappedField;
  204. });
  205. return mappedFields;
  206. }
  207. private mapFieldType(dataType: string): FieldTypeEnum {
  208. const type = dataType?.toLowerCase();
  209. const mappedType = (() => {
  210. switch (type) {
  211. case 'time':
  212. return FieldTypeEnum.Time;
  213. case 'decimal':
  214. case 'number':
  215. case 'float':
  216. return FieldTypeEnum.Decimal;
  217. case 'date':
  218. return FieldTypeEnum.Date;
  219. case 'int':
  220. case 'integer':
  221. case 'numeric':
  222. return FieldTypeEnum.Numeric;
  223. default:
  224. return FieldTypeEnum.String;
  225. }
  226. })();
  227. return mappedType;
  228. }
  229. private convertCellValue(dataType: string, dataTypeFormat: string | undefined, value: any, fieldType: FieldTypeEnum): any {
  230. if (value === null || value === undefined) {
  231. return null;
  232. }
  233. const convertedValue = (() => {
  234. if (dataType === "DATE" && (typeof value === 'number' || value instanceof Date))
  235. return XLSX.SSF.format(dataTypeFormat || 'yyyy-mm-dd', value);
  236. switch (fieldType) {
  237. case FieldTypeEnum.Time:
  238. if (typeof value === 'number') {
  239. // Excel time is fraction of a day
  240. const result = value * 24 * 60 * 60 * 1000; // Convert to milliseconds
  241. return result;
  242. }
  243. return value;
  244. case FieldTypeEnum.Decimal:
  245. const decimalResult = parseFloat(value.toString()) || 0;
  246. return decimalResult;
  247. case FieldTypeEnum.Date:
  248. if (typeof value === 'number') {
  249. // Excel date is days since 1900-01-01
  250. const excelEpoch = new Date(1900, 0, 1);
  251. const dateResult = new Date(excelEpoch.getTime() + (value - 1) * 24 * 60 * 60 * 1000);
  252. return dateResult;
  253. }
  254. const dateResult = new Date(value);
  255. return dateResult;
  256. case FieldTypeEnum.Numeric:
  257. const numericResult = parseInt(value.toString()) || 0;
  258. return numericResult;
  259. case FieldTypeEnum.String:
  260. default:
  261. const stringResult = value.toString();
  262. return stringResult;
  263. }
  264. })();
  265. return convertedValue;
  266. }
  267. }