Working header correction. Reword excel parsing/matching. Ready to submit data to server

associator_rewrite
Griffiths Lott 3 years ago
parent 1d8d92d595
commit 6e59501a2d
  1. 3
      babel.config.js
  2. 30
      package-lock.json
  3. 1
      package.json
  4. 13
      src/App.vue
  5. 107
      src/components/FileUpload.vue
  6. 42
      src/components/InvalidValues.vue
  7. 69
      src/components/ItemBlock.vue
  8. 80
      src/components/ItemReplace.vue
  9. 6
      src/components/Missing.vue
  10. 52
      src/helpers.js
  11. 16
      src/store.js

@ -1,5 +1,8 @@
module.exports = {
presets: [
'@vue/cli-plugin-babel/preset'
],
plugins: [
'@babel/plugin-proposal-private-methods'
]
}

30
package-lock.json generated

@ -10,6 +10,7 @@
"dependencies": {
"core-js": "^3.8.3",
"vue": "^3.2.13",
"vuex": "^4.1.0",
"xlsx": "https://cdn.sheetjs.com/xlsx-0.19.1/xlsx-0.19.1.tgz"
},
"devDependencies": {
@ -2914,6 +2915,11 @@
"integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==",
"dev": true
},
"node_modules/@vue/devtools-api": {
"version": "6.5.0",
"resolved": "https://registry.npmjs.org/@vue/devtools-api/-/devtools-api-6.5.0.tgz",
"integrity": "sha512-o9KfBeaBmCKl10usN4crU53fYtC1r7jJwdGKjPT24t348rHxgfpZ0xL3Xm/gLUYnc0oTp8LAmrxOeLyu6tbk2Q=="
},
"node_modules/@vue/reactivity": {
"version": "3.2.45",
"resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.2.45.tgz",
@ -10789,6 +10795,17 @@
"integrity": "sha512-4gDntzrifFnCEvyoO8PqyJDmguXgVPxKiIxrBKjIowvL9l+N66196+72XVYR8BBf1Uv1Fgt3bGevJ+sEmxfZzw==",
"dev": true
},
"node_modules/vuex": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/vuex/-/vuex-4.1.0.tgz",
"integrity": "sha512-hmV6UerDrPcgbSy9ORAtNXDr9M4wlNP4pEFKye4ujJF8oqgFFuxDCdOLS3eNoRTtq5O3hoBDh9Doj1bQMYHRbQ==",
"dependencies": {
"@vue/devtools-api": "^6.0.0-beta.11"
},
"peerDependencies": {
"vue": "^3.2.0"
}
},
"node_modules/watchpack": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz",
@ -13697,6 +13714,11 @@
}
}
},
"@vue/devtools-api": {
"version": "6.5.0",
"resolved": "https://registry.npmjs.org/@vue/devtools-api/-/devtools-api-6.5.0.tgz",
"integrity": "sha512-o9KfBeaBmCKl10usN4crU53fYtC1r7jJwdGKjPT24t348rHxgfpZ0xL3Xm/gLUYnc0oTp8LAmrxOeLyu6tbk2Q=="
},
"@vue/reactivity": {
"version": "3.2.45",
"resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.2.45.tgz",
@ -19559,6 +19581,14 @@
"integrity": "sha512-4gDntzrifFnCEvyoO8PqyJDmguXgVPxKiIxrBKjIowvL9l+N66196+72XVYR8BBf1Uv1Fgt3bGevJ+sEmxfZzw==",
"dev": true
},
"vuex": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/vuex/-/vuex-4.1.0.tgz",
"integrity": "sha512-hmV6UerDrPcgbSy9ORAtNXDr9M4wlNP4pEFKye4ujJF8oqgFFuxDCdOLS3eNoRTtq5O3hoBDh9Doj1bQMYHRbQ==",
"requires": {
"@vue/devtools-api": "^6.0.0-beta.11"
}
},
"watchpack": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz",

@ -10,6 +10,7 @@
"dependencies": {
"core-js": "^3.8.3",
"vue": "^3.2.13",
"vuex": "^4.1.0",
"xlsx": "https://cdn.sheetjs.com/xlsx-0.19.1/xlsx-0.19.1.tgz"
},
"devDependencies": {

@ -1,15 +1,26 @@
<template>
<FileUpload></FileUpload>
<ItemBlock top-message="Missing Headers:" :item-list="store.missingHeaders"></ItemBlock>
</template>
// <ItemBlock top-message="Invalid Data in Columns:" :item-list="store.documentHeaders.filter(aHeader => !aHeader.isSet())"></ItemBlock>
<script>
//import HelloWorld from './components/HelloWorld.vue'
import FileUpload from './components/FileUpload.vue';
import ItemBlock from './components/ItemBlock.vue';
import { store } from './store';
export default {
name: 'App',
components: {
FileUpload
FileUpload,
ItemBlock,
},
data() {
return {
store
}
}
}
</script>

@ -1,3 +1,4 @@
// TODO Add submit button to link with file upload
<template>
<form>
<input type="file" id="uploadInput" accept=".xlsx" @change="upload($event)">
@ -7,6 +8,8 @@
<script>
import { read, utils } from "xlsx";
import { store } from '../store.js'
import { HeaderAssociator } from '../helpers.js'
function find_header_row(worksheet) {
// Returns the zero-index row number of the headers
@ -35,12 +38,13 @@ export default {
uploadedFiles: [],
uploadError: null,
currentStatus: null,
uploadFieldName: 'uploadfile'
uploadFieldName: 'uploadfile',
store
}
},
methods: {
async upload(event) {
var file = await event.target.files[0].arrayBuffer();
var workbook = read(file);
var worksheets = workbook.SheetNames;
@ -49,28 +53,28 @@ export default {
var headerIndex = find_header_row(worksheet);
// TODO: Add handling for failed uploads and checking other sheets
// FIXME: Add handling for failed uploads and checking other sheets
if (headerIndex === null) {
console.log("Failed to find valid header row");
console.error("Failed to find valid header row");
return;
}
// TODO: This should happen on page load and be stored as app data
// Load header/value config
var configRequestOptions = {
method: 'GET',
redirect: 'follow',
};
var resp = await fetch("http://192.168.0.171:5252/api/template", configRequestOptions);
var dataStandardList = await resp.json();
console.log(dataStandardList);
console.info(dataStandardList);
// Here we compile all of the headers into one string
// This will let us easily search for each header using regex
// The col will be noted at the end of the header so that we can
// check the value
var headers = "";
var combinedHeadersString = "";
console.group("Document columns");
for (let col = 0; col < 200; col++){
let cellAddress = utils.encode_cell({c: col, r: headerIndex});
@ -78,45 +82,64 @@ export default {
if (cellObj === undefined) { break }
// A value of 40k typicall indicates a date (cashflows)
else if (cellObj.v > 40000) { break }
// Add [ col # ] to help find value in the same column
headers = headers + worksheet[cellAddress].v + '[' + col + ']';
// Add the header to the combined headers string
combinedHeadersString += cellObj.v;
let headerAssociator = new HeaderAssociator(cellObj.v, cellAddress);
console.info(`Document column: ${headerAssociator}`)
store.documentHeaders.push(headerAssociator);
}
console.groupEnd();
var missing = new Array();
var invalidData = new Array();
var colIndexReg = RegExp("\\[\\d{1,3}\\]");
for (let i in dataStandardList) {
let headerRegex = dataStandardList[i];
// Check each template regex against the combined headers string
for (const headerTemplate of dataStandardList) {
console.group(`Header regex: ${headerTemplate["Header"]}`);
// 'i' option means ignore case
// ADding [\d{1,3}] allows us to find the column the match came from
let searchRegex = RegExp(headerRegex["HeaderRegex"]+"\\[\\d{1,3}\\]","i");
let found = searchRegex.exec(headers);
if (found === null) { missing.push(headerRegex); continue}
// Now we need to check the value below
let isNullable = headerRegex["Nullable"];
// Get the value in the space
let firstMatch = found[0];
let colInd = firstMatch.match(colIndexReg)[0];
colInd = parseInt(colInd.replace('[','').replace(']',''));
// Now we can get the value of the cell below the header
let valueCellAddress = utils.encode_cell({c: colInd, r: headerIndex+1});
let valueObj = worksheet[valueCellAddress];
// Test if there is anything here, if not this is a invald
if (valueObj === undefined ) {
if (isNullable) {continue}
invalidData.push(headerRegex); continue
let headerRegex = RegExp(headerTemplate["HeaderRegex"],"i");
let headerName = headerTemplate["Header"];
let searchResults = headerRegex.exec(combinedHeadersString);
if (searchResults === null) {
console.info(`Header not found: ${headerName}`);
store.missingHeaders.push(headerName);
console.groupEnd()
continue;
}
// Test the value against regex
searchRegex = RegExp(headerRegex["ValueRegex"],'i');
let validData = searchRegex.test(valueObj.v);
if (!validData) { invalidData.push(headerRegex)}
let docHeader = searchResults[0];
console.log(`Match: ${docHeader}`);
// Now we need to check the value below the header
let isNullable = headerTemplate["Nullable"];
// Find the associated header
for (let headerAssociator of store.documentHeaders) {
if (headerAssociator.isSet()) {console.groupEnd();continue}
if (headerAssociator.documentHeader === docHeader) {
headerAssociator.swapTemplateHeader(headerName);
let headerAddress = utils.decode_cell(headerAssociator.docHeaderAddress);
// Now we can get the value of the cell below the header
let valueCellAddress = utils.encode_cell(headerAddress.r + 1);
let valueObj = worksheet[valueCellAddress];
// Check if the cell is null, if so does it matter?
if (valueObj !== undefined ) {
// Test the value against regex
let valueRegex = RegExp(headerTemplate["ValueRegex"],'i');
headerAssociator.isValidData = valueRegex.test(valueObj.v);
} else {
headerAssociator.isValidData = isNullable;
}
console.info(`Found header: ${headerAssociator}`);
console.groupEnd();
break;
}
}
console.groupEnd();
}
console.log(missing);
console.log(invalidData);
console.info(`Missing headers (${store.missingHeaders.length}): ${store.missingHeaders}`);
console.info(`Document headers (${store.documentHeaders.length}): ${store.documentHeaders}`);
// let excelDoc = new FormData();
// excelDoc.append("file", event.target.files[0]);

@ -0,0 +1,42 @@
<template>
<div v-if="store.missingHeaders.length > 0" class="missing-headers">
<h1>Missing Headers:</h1>
<div class="grid">
<ColumnHeader v-for="header in store.missingHeaders" :key="header" :text="header"/>
</div>
</div>
</template>
<script>
import { store } from '../store.js'
export default {
components: {
},
data() {
return {
store
}
},
}
</script>
<style>
.missing-headers {
text-align: center;
}
.grid {
justify-content: center;
display: grid;
grid-template-columns: repeat(3, 1fr);
grid-gap: 10px;
margin-top: 10px;
padding: 15px;
border-color: darkgrey;
border-width: 10px;
background-color:#800000;
}
</style>

@ -0,0 +1,69 @@
<template>
<div v-if="store.missingHeaders.length > 0" class="missing-headers" :style="isFixed ? {'background-color': '#0a6e11'} : {'background-color': '#990b0b'}" >
<h1>{{topMessage}}</h1>
<div class="grid" :style="isFixed ? {'background-color': '#0b3b0e'} : {'background-color': '#800000'}">
<ItemReplace @header-change="itemChanged" v-for="header in itemList" :key="header" :itemText="header"/>
</div>
<button v-if="this.isFixed" class="submit-button" @click="submitCorrections">Submit Corrections</button>
</div>
</template>
<script>
import { store } from '../store.js'
import ItemReplace from './ItemReplace.vue'
export default {
props: {
itemList: Array,
topMessage: String
},
data() {
return {
store,
correctedColumns : 0,
isFixed : false,
}
},
components: {
ItemReplace
},
methods: {
itemChanged(value) {
console.log("Value: " + value + " itemlistlen: "+ this.itemList.length)
this.correctedColumns += value;
this.isFixed = Boolean(this.correctedColumns == this.itemList.length)
console.log("isFixed: " + this.isFixed)
},
submitCorrections() {
console.log("Correction Submitted!")
}
}
}
</script>
<style>
.missing-headers {
text-align: center;
background-color: #ac3232;
}
.grid {
justify-content: center;
display: grid;
grid-template-columns: repeat(3, 1fr);
grid-gap: 10px;
margin-top: 10px;
padding: 15px;
border-color: darkgrey;
border-width: 10px;
}
.submit-button{
text-align: center;
margin-top: 20px;
font-size: large;
}
</style>

@ -0,0 +1,80 @@
<template>
<div class="box" :style="{'background-color': this.isConfirmed ? '#078f12' : '#ff4242'}">
<div class="box-header">
<h3 class="text-center">{{itemText}}</h3>
</div>
<div class="box-body">
<select class="form-control" @change="updateSelectedDocHeader($event)">
<option>{{firstValue}}</option>
<option v-for="headerObj in filteredHeaders" :key="headerObj.header.documentHeader" :value="headerObj.index">{{headerObj.header.documentHeader}}</option>
</select>
<button type="button" class="btn btn-primary" @click="changeA" :disabled="this.selectedDocHeaderIndex === null">Confirm Choice</button>
</div>
</div>
</template>
<script>
import { store } from '@/store';
export default {
props: {
itemText: {
type: String,
required: true
},
},
data() {
return {
backgroundColor: "#ff6666",
selectedDocHeaderIndex: null,
isConfirmed: false,
store
}
},
methods: {
updateSelectedDocHeader(event) {
if (this.isConfirmed) { this.$emit('header-change', -1); }
this.isConfirmed = false;
// If the old index is not null, we need to update it's docHeader to be null
if (this.selectedDocHeaderIndex!== null) {
this.store.documentHeaders[this.selectedDocHeaderIndex].swapTemplateHeader(null);
}
this.selectedDocHeaderIndex = event.target.value;
console.log(this.selectedDocHeaderIndex);
},
changeA() {
// Change the templateHeader in the docHeader at selectedDocHeader
this.isConfirmed = true;
this.store.documentHeaders[this.selectedDocHeaderIndex].templateHeader = this.store.documentHeaders;
this.$emit('header-change', 1);
}
},
computed: {
filteredHeaders() {
return this.store.documentHeaders.filter(header => {
return !header.isSet();
}).map(header => {
return {
index: this.store.documentHeaders.indexOf(header),
header: header,
};
});
},
firstValue() {
return this.isConfirmed ? this.store.documentHeaders[this.selectedDocHeaderIndex].documentHeader : null
}
}
}
</script>
<style>
.box {
width: 100%;
height: 100%;
border-radius: 10px;
text-align: center;
}
</style>

@ -1,6 +0,0 @@
<template>
<form>
<input type="file" id="uploadInput" accept=".xlsx" @change="upload($event)">
</form>
</template>

@ -0,0 +1,52 @@
export class HeaderAssociator {
/*
This class is used to track the headers of the excel file
uploaded by the user. The header can then be assoicate with a
header from our specified template. The cell address of the
document header is also tracked.
*/
// The users excel header they want to associate with our template
// type: (string)
documentHeader;
// The excel address of the document header
// This will be a string in A1 notation for Excel
docHeaderAddress;
// The template excel header they want to associate
// the document header with
// type: (string)
templateHeader = null;
isValidData = false;
constructor(documentHeader, address) {
this.documentHeader = documentHeader
this.docHeaderAddress = address
}
set isValidData(newValidData) {
if (typeof newValidData !== 'boolean') {
console.error(`${this.documentHeader} | HeaderAssociator.validData must be a boolean: ${newValidData}`);
this.isValidData = false;
}
else this.isValidData = newValidData;
}
swapTemplateHeader(newTemplateHeader) {
// Set the template header and return the previous template header
// May return null if nothing has been set
let oldTemplateHeader = this.templateHeader
// Convert blank string to null
this.templateHeader = newTemplateHeader === '' ? null : newTemplateHeader
return oldTemplateHeader
}
toString(){
return `${this.documentHeader} (${this.docHeaderAddress}) : ${this.templateHeader}`
}
isSet() {
return this.templateHeader !== null
}
}

@ -0,0 +1,16 @@
// store.js
import { reactive } from 'vue'
export const store = reactive({
// Template headers missing from the document (strings)
// regex did not match on any headers found in the document
missingHeaders: [],
// Headers found by scanning the document (HeaderAssociator)
// Contains the following data:
// - docuemnt header
// - document header address (excel A1 string)
// - template header (or null)
// - isValidDate : whether the excel row data is valid
documentHeaders: [],
})
Loading…
Cancel
Save