associator_rewrite
parent
0d29079c30
commit
72113dc863
@ -0,0 +1,230 @@ |
|||||||
|
<template> <template> |
||||||
|
<form> <form> |
||||||
|
<input type="file" id="uploadInput" accept=".xlsx" @chang <input type="file" id="uploadInput" accept=".xlsx" @chang |
||||||
|
<button type="submit" @click.prevent="parseUpload" :disab <button type="submit" @click.prevent="parseUpload" :disab |
||||||
|
</form> </form> |
||||||
|
</template> </template> |
||||||
|
|
||||||
|
<script> <script> |
||||||
|
|
||||||
|
import { read, utils } from "xlsx"; import { read, utils } from "xlsx"; |
||||||
|
import { store } from '../store.js' | import { store, clearStore } from '../store.js' |
||||||
|
import { HeaderAssociator } from '../helpers.js' | import { UserHeader } from "@/userHeader"; |
||||||
|
> import { HeaderTemplate, associateHeaders } from "@/headerTem |
||||||
|
> |
||||||
|
|
||||||
|
function findFirstNonBlankRow(worksheet) { function findFirstNonBlankRow(worksheet) { |
||||||
|
const range = utils.decode_range(worksheet['!ref']); const range = utils.decode_range(worksheet['!ref']); |
||||||
|
// We'll check at most of 20 rows // We'll check at most of 20 rows |
||||||
|
for (let row = range.s.r; row <= Math.min(range.e.r,20); ro for (let row = range.s.r; row <= Math.min(range.e.r,20); ro |
||||||
|
var headerRow = true; var headerRow = true; |
||||||
|
for (let col = 0; col < 20; col++) { for (let col = 0; col < 20; col++) { |
||||||
|
const cell = worksheet[utils.encode_cell({ r: row, c: c const cell = worksheet[utils.encode_cell({ r: row, c: c |
||||||
|
if (!cell || cell.v === '') { if (!cell || cell.v === '') { |
||||||
|
headerRow = false; headerRow = false; |
||||||
|
break; break; |
||||||
|
} } |
||||||
|
} } |
||||||
|
if (headerRow) { if (headerRow) { |
||||||
|
return row; return row; |
||||||
|
} } |
||||||
|
} } |
||||||
|
return null; return null; |
||||||
|
} } |
||||||
|
|
||||||
|
|
||||||
|
export default { export default { |
||||||
|
name: 'FileUpload', name: 'FileUpload', |
||||||
|
data() { data() { |
||||||
|
return { return { |
||||||
|
// User uploaded file // User uploaded file |
||||||
|
userFile: null, userFile: null, |
||||||
|
// Header Template provided by server // Header Template provided by server |
||||||
|
headerTemplateList: [], | headerTemplateList: null, |
||||||
|
> userHeaders: [], |
||||||
|
// Defines whether data row is checked agianst value re // Defines whether data row is checked agianst value re |
||||||
|
validateData: true, | invalidData: [], |
||||||
|
// Store data for access by other components // Store data for access by other components |
||||||
|
store, store, |
||||||
|
// Controls whether or not the user can submit a file // Controls whether or not the user can submit a file |
||||||
|
// - When a file is selected, then disabled when submit // - When a file is selected, then disabled when submit |
||||||
|
canSubmit: false, canSubmit: false, |
||||||
|
} } |
||||||
|
}, }, |
||||||
|
async mounted() { async mounted() { |
||||||
|
// We want to get the lastest template header specificati // We want to get the lastest template header specificati |
||||||
|
try { try { |
||||||
|
const configRequestOptions = { const configRequestOptions = { |
||||||
|
method: 'GET', method: 'GET', |
||||||
|
redirect: 'follow', redirect: 'follow', |
||||||
|
}; }; |
||||||
|
const resp = await fetch(process.env.VUE_APP_TEMPLATE_H const resp = await fetch(process.env.VUE_APP_TEMPLATE_H |
||||||
|
let respJson = await resp.json(); let respJson = await resp.json(); |
||||||
|
console.log(respJson); | console.info("RespJ: ", respJson); |
||||||
|
this.headerTemplateList = respJson["HeaderTemplate"]; < |
||||||
|
//FIXME: This is only for testing purposes, REMOVE THIS //FIXME: This is only for testing purposes, REMOVE THIS |
||||||
|
this.validateData = true;//respJson["ValidateData"]; this.validateData = true;//respJson["ValidateData"]; |
||||||
|
console.log(this.headerTemplateList); | this.headerTemplateList = respJson["HeaderTemplate"].ma |
||||||
|
console.log(this.validateData); | return new HeaderTemplate (headerTemplate["Header"],h |
||||||
|
> this.validateData, headerTemplate["ValueRegex"], he |
||||||
|
> }); |
||||||
|
> // Sort this list to make association faster |
||||||
|
> this.headerTemplateList.sort((a,b) => { |
||||||
|
> const ha = a.header.toLowerCase(); |
||||||
|
> const hb = b.header.toLowerCase(); |
||||||
|
> if (ha < hb) return -1; |
||||||
|
> if (ha > hb) return 1; |
||||||
|
> return 0; |
||||||
|
> }); |
||||||
|
> console.info("Recieved HeaderTemplateList: ", this.head |
||||||
|
} catch (error) { } catch (error) { |
||||||
|
//FIXME: emit error internal issues //FIXME: emit error internal issues |
||||||
|
console.error(error); console.error(error); |
||||||
|
} } |
||||||
|
}, }, |
||||||
|
methods: { methods: { |
||||||
|
|
||||||
|
fileSelected(event) { fileSelected(event) { |
||||||
|
// Make sure the user can't still act on the old file // Make sure the user can't still act on the old file |
||||||
|
this.$emit('parsed-upload', null); | this.$emit('parsed-upload', null, []); |
||||||
|
// Clear old file data // Clear old file data |
||||||
|
store.missingHeaders = []; | this.invalidData = []; |
||||||
|
store.documentHeaders = []; | this.userHeaders = []; |
||||||
|
> clearStore(); |
||||||
|
> |
||||||
|
> // If the user had previously selected a file, |
||||||
|
> // we need to clear the old header associations |
||||||
|
> if (this.userFile !== null) { |
||||||
|
> this.headerTemplateList.map(headerTemplate => { |
||||||
|
> headerTemplate.userHeader = null; |
||||||
|
> }) |
||||||
|
> } |
||||||
|
> |
||||||
|
|
||||||
|
this.userFile = event.target.files[0]; this.userFile = event.target.files[0]; |
||||||
|
this.canSubmit = true; this.canSubmit = true; |
||||||
|
}, }, |
||||||
|
|
||||||
|
getHeaders(worksheet, sheetName, headerIndex) { getHeaders(worksheet, sheetName, headerIndex) { |
||||||
|
// Store the found headers // Store the found headers |
||||||
|
var combinedHeadersString = ""; < |
||||||
|
// Use the range to make sure we don't go out of bounds // Use the range to make sure we don't go out of bounds |
||||||
|
const range = utils.decode_range(worksheet['!ref']); const range = utils.decode_range(worksheet['!ref']); |
||||||
|
for (let col = 0; col < range.e.c; col++){ for (let col = 0; col < range.e.c; col++){ |
||||||
|
let cellAddress = utils.encode_cell({c: col, r: heade let cellAddress = utils.encode_cell({c: col, r: heade |
||||||
|
let cellObj = worksheet[cellAddress]; let cellObj = worksheet[cellAddress]; |
||||||
|
// n represents number type (which is probably the st // n represents number type (which is probably the st |
||||||
|
if (cellObj === undefined || cellObj.t === 'n') { bre if (cellObj === undefined || cellObj.t === 'n') { bre |
||||||
|
// Add the header to the combined headers string | let dataAddress = utils.encode_cell({c: col, r: heade |
||||||
|
combinedHeadersString += cellObj.v; | let dataValue = worksheet[dataAddress] ? worksheet[da |
||||||
|
let headerAssociator = new HeaderAssociator(cellObj.v | |
||||||
|
if (!this.validateData) {headerAssociator.isValidData | // Record the user header |
||||||
|
store.documentHeaders.push(headerAssociator); | let userHeader = new UserHeader(cellObj.v, sheetName, |
||||||
|
> this.userHeaders.push(userHeader); |
||||||
|
} } |
||||||
|
return combinedHeadersString; < |
||||||
|
}, }, |
||||||
|
|
||||||
|
async parseUpload() { async parseUpload() { |
||||||
|
this.canSubmit = false; this.canSubmit = false; |
||||||
|
const fileBuffer = await this.userFile.arrayBuffer(); const fileBuffer = await this.userFile.arrayBuffer(); |
||||||
|
var workbook = await read(fileBuffer, {type: 'array'}); var workbook = await read(fileBuffer, {type: 'array'}); |
||||||
|
|
||||||
|
if (workbook === null) { if (workbook === null) { |
||||||
|
//FIXME: Emit bad spreadsheet to app //FIXME: Emit bad spreadsheet to app |
||||||
|
console.error(`No workbook found! Could not parse fil console.error(`No workbook found! Could not parse fil |
||||||
|
return; return; |
||||||
|
} } |
||||||
|
|
||||||
|
var validWorksheets = [] | let validWorksheets = false; |
||||||
|
var combinedHeadersString = ''; < |
||||||
|
// Go through each worksheet // Go through each worksheet |
||||||
|
for (let sheetName of workbook.SheetNames) { for (let sheetName of workbook.SheetNames) { |
||||||
|
// Open the actual worksheet // Open the actual worksheet |
||||||
|
let worksheet = workbook.Sheets[sheetName]; let worksheet = workbook.Sheets[sheetName]; |
||||||
|
// Find the first non-blank row of 20 // Find the first non-blank row of 20 |
||||||
|
var headerIndex = findFirstNonBlankRow(worksheet); var headerIndex = findFirstNonBlankRow(worksheet); |
||||||
|
if (headerIndex === null) { if (headerIndex === null) { |
||||||
|
// Probably not a sheet we need to process // Probably not a sheet we need to process |
||||||
|
console.warn(`${sheetName} | No header row found!`) console.warn(`${sheetName} | No header row found!`) |
||||||
|
} } |
||||||
|
// Add the header to the combined headers string | // getHeaders will created the UserHeader objects |
||||||
|
// getHeaders will created the AssociatedHeaders obje | this.getHeaders(worksheet, sheetName, headerIndex); |
||||||
|
combinedHeadersString += this.getHeaders(worksheet, s | validWorksheets = true; |
||||||
|
// Note which worksheets are actually being processed < |
||||||
|
validWorksheets.push(sheetName); < |
||||||
|
} } |
||||||
|
if (validWorksheets.length === 0) { | if (!validWorksheets) { |
||||||
|
//FIXME: emit bad spreadsheet to App //FIXME: emit bad spreadsheet to App |
||||||
|
console.error("No worksheets found!"); console.error("No worksheets found!"); |
||||||
|
return; return; |
||||||
|
} } |
||||||
|
|
||||||
|
// Check each template regex against the combined heade | // Sort the user headers to make association faster |
||||||
|
for (const headerTemplate of this.headerTemplateList) { | this.userHeaders.sort((a,b) => { |
||||||
|
// 'i' option means ignore case | const ha = a.label.toLowerCase(); |
||||||
|
let headerRegex = RegExp(headerTemplate["HeaderRegex" | const hb = b.label.toLowerCase(); |
||||||
|
let headerName = headerTemplate["Header"]; | if (ha < hb) return -1; |
||||||
|
| if (ha > hb) return 1; |
||||||
|
// Check to see if the template header is present in | return 0; |
||||||
|
let searchResults = headerRegex.exec(combinedHeadersS | }); |
||||||
|
if (searchResults === null) { | |
||||||
|
store.missingHeaders.push(headerName); | // console.log("User Headers:", this.userHeaders); |
||||||
|
continue; | // console.log("Template Headers:", this.headerTemplate |
||||||
|
} | let processedArrays = associateHeaders(this.userHeaders |
||||||
|
let docHeader = searchResults[0]; | |
||||||
|
> store.unassociatedUserHeaders = processedArrays[0]; |
||||||
|
> store.unassociatedTemplateHeaders = this.headerTemplate |
||||||
|
> !headerTemplate.isAssociated(); |
||||||
|
> }); |
||||||
|
> this.invalidData = processedArrays[1]; |
||||||
|
> console.log("Invalid Data:", this.invalidData); |
||||||
|
> console.log("Unassociated:", store.unAssociated); |
||||||
|
> |
||||||
|
> console.log("Updated Template Headers:", this.headerTem |
||||||
|
|
||||||
|
// Find the associated header < |
||||||
|
for (let headerAssociator of store.documentHeaders) { < |
||||||
|
if (headerAssociator.isSet()) {continue} < |
||||||
|
if (headerAssociator.documentHeader === docHeader) < |
||||||
|
// Set the associated template header < |
||||||
|
headerAssociator.swapTemplateHeader(headerName); < |
||||||
|
< |
||||||
|
if (this.validateData) { < |
||||||
|
// If we want to validate the data, then check < |
||||||
|
let isNullable = headerTemplate["Nullable"]; < |
||||||
|
let headerAddress = utils.decode_cell(headerAss < |
||||||
|
// Now we can get the value of the cell below t < |
||||||
|
let valueCellAddress = utils.encode_cell(header < |
||||||
|
// We need to know which worksheet this cell be < |
||||||
|
let worksheet = workbook.Sheets[headerAssociato < |
||||||
|
let valueObj = worksheet[valueCellAddress]; < |
||||||
|
// Check if the cell is null, if so does it mat < |
||||||
|
if (valueObj !== undefined ) { < |
||||||
|
// Test the value against regex < |
||||||
|
let valueRegex = RegExp(headerTemplate["Value < |
||||||
|
let dataValue = valueObj.v.toString(); < |
||||||
|
let isValid = valueRegex.test(dataValue); < |
||||||
|
if (!isValid) { < |
||||||
|
// record the invalid data and set isValidD < |
||||||
|
headerAssociator.invalidData(dataValue); < |
||||||
|
} < |
||||||
|
} else { < |
||||||
|
// isValid matches isNullable < |
||||||
|
headerAssociator.isValidData = isNullable; < |
||||||
|
} < |
||||||
|
} < |
||||||
|
// We found the header and processed it so stop l < |
||||||
|
break; < |
||||||
|
} < |
||||||
|
} < |
||||||
|
} < |
||||||
|
//TODO: Remove logs < |
||||||
|
console.log(`Document headers (${store.documentHeaders. < |
||||||
|
console.log(`Missing headers (${store.missingHeaders.le < |
||||||
|
// emit the uploaded file & parsed data up to parent // emit the uploaded file & parsed data up to parent |
||||||
|
this.$emit('parsed-upload', this.userFile); | this.$emit('parsed-upload', this.userFile, this.invalid |
||||||
|
} } |
||||||
|
} } |
||||||
|
} } |
||||||
|
</script> </script> |
||||||
Loading…
Reference in new issue