Skip to content

T
topjs3 study
Commit 4372cebd authored by Scott Sun's avatar Scott Sun
Browse files
Options

s

parent b4045894
Show whitespace changes
Inline Side-by-side
Showing with 179 additions and 50 deletions
2019-12-31/scott/README.md 0 → 100644
View file @ 4372cebd
## config.json说明
* dirPath 存放xlsx的文件夹路径
* isUseing 判断脚本是否在运行 如果正在运行 则不执行脚本
* rdir 读取过的xlsx会剪切到这里
* dbconfig 数据库的配置
* dataKeys 数据库的字段 key是excel读取到的表的md5加密 值是对象 值的name是对应的字段名 dbkey是数据库对应的字段名
* attr_data是特殊字段 存放新增的excel表字段
* 如果运行不了 可能是某些错误导致isUseing未及时变成未运行状态 所以这种情况手动将isUseing改为false
2019-12-31/scott/config.json
View file @ 4372cebd
{"dirPath":"../xlsx","isUseing":false,"rdir":"../rxlsx","dbconfig":{"database_type":"pg","database_host":"139.196.104.13:5433","database_name":"PMO_TRAINING","database_user":"toplinker","database_pwd":"TopLinker0510"},"keys":["order_no","confirmation","plant","material","material_description","posting_date","confirmation_entry_time","activity","work_center","confirmed_yield","base_unit","confirmed_scrap","reason_for_variance","rework","confirmation_text","entered_by_user","ind_final_confirmation","milestone_confirmed","ind_delete_doc"],"dataKeys":{"135aa36437beb6571531ca8e898ad57a":{"name":"Order\n生管投料单号","dbkey":"order_no"},"a1deefe94c73d05d2a3b247c23362042":{"name":"Confirmation\n总数量","dbkey":"confirmation"},"c46b4eb1d67a08b863072b33b646a810":{"name":"Plant\n厂别","dbkey":"plant"},"e5018abee57b7f610ef8bb4d9eecc112":{"name":"Material\n批号","dbkey":"material"},"f19e926ffab820b96d5a6aabf9aae616":{"name":"Material Description\n批次说明","dbkey":"material_description"},"7e51e12322207097a84891f9d7445ad4":{"name":"Posting Date\n生产时间","dbkey":"posting_date"},"8dba6faf3484a91f3b3940bb2e9ea459":{"name":"Confirmation Entry Time\n确认时间","dbkey":"confirmation_entry_time"},"ecfc2dffe568c10a67dbc6d3724cfde2":{"name":"Activity","dbkey":"activity"},"657a1d49f3e92290e56b692ff58a0a2d":{"name":"Work Center\n生产站别","dbkey":"work_center"},"62f5f57f24ea1d0bc12aa535d6add05d":{"name":"Confirmed Yield (GMEIN)","dbkey":"confirmed_yield"},"7c8590875b72ce8eca3c92f76b2c451b":{"name":"Base Unit of Measure (=GMEIN)","dbkey":"base_unit"},"37c9b123cc9f488d3f7f8480146795e9":{"name":"Confirmed scrap (MEINH)\n报废数量","dbkey":"confirmed_scrap"},"91b8b2e603be16c0a94658f0ced49c0b":{"name":"Reason for Variance\n报废代码","dbkey":"reason_for_variance"},"55979da7be6e74a5ce14bd81cd31028d":{"name":"Rework\n重工数量","dbkey":"rework"},"e820bdb5f699f28fb15c5d5e759f164b":{"name":"Confirmation text","dbkey":"confirmation_text"},"da01091b1f5ddb363d498e844449a961":{"name":"Entered by User","dbkey":"entered_by_user"},"664152bb1a151874f332b55ec47bc54d":{"name":"Ind.Final Confirmation","dbkey":"ind_final_confirmation"},"196c6ed505a0dabe34f0aed4e2800535":{"name":"Milestone confirmed","dbkey":"milestone_confirmed"},"0b9bf2967b86c25513ff839bcf3eab83":{"name":"Ind. delete doc.","dbkey":"ind_delete_doc"}}}
\ No newline at end of file
{"dirPath":"../xlsx","isUseing":false,"rdir":"../rxlsx","dbconfig":{"database_type":"pg","database_host":"139.196.104.13:5433","database_name":"PMO_TRAINING","database_user":"toplinker","database_pwd":"TopLinker0510"},"dataKeys":{"135aa36437beb6571531ca8e898ad57a":{"name":"Order\n生管投料单号","dbkey":"order_no"},"a1deefe94c73d05d2a3b247c23362042":{"name":"Confirmation\n总数量","dbkey":"confirmation"},"c46b4eb1d67a08b863072b33b646a810":{"name":"Plant\n厂别","dbkey":"plant"},"e5018abee57b7f610ef8bb4d9eecc112":{"name":"Material\n批号","dbkey":"material"},"f19e926ffab820b96d5a6aabf9aae616":{"name":"Material Description\n批次说明","dbkey":"material_description"},"7e51e12322207097a84891f9d7445ad4":{"name":"Posting Date\n生产时间","dbkey":"posting_date"},"8dba6faf3484a91f3b3940bb2e9ea459":{"name":"Confirmation Entry Time\n确认时间","dbkey":"confirmation_entry_time"},"ecfc2dffe568c10a67dbc6d3724cfde2":{"name":"Activity","dbkey":"activity"},"657a1d49f3e92290e56b692ff58a0a2d":{"name":"Work Center\n生产站别","dbkey":"work_center"},"62f5f57f24ea1d0bc12aa535d6add05d":{"name":"Confirmed Yield (GMEIN)","dbkey":"confirmed_yield"},"7c8590875b72ce8eca3c92f76b2c451b":{"name":"Base Unit of Measure (=GMEIN)","dbkey":"base_unit"},"37c9b123cc9f488d3f7f8480146795e9":{"name":"Confirmed scrap (MEINH)\n报废数量","dbkey":"confirmed_scrap"},"91b8b2e603be16c0a94658f0ced49c0b":{"name":"Reason for Variance\n报废代码","dbkey":"reason_for_variance"},"55979da7be6e74a5ce14bd81cd31028d":{"name":"Rework\n重工数量","dbkey":"rework"},"e820bdb5f699f28fb15c5d5e759f164b":{"name":"Confirmation text","dbkey":"confirmation_text"},"da01091b1f5ddb363d498e844449a961":{"name":"Entered by User","dbkey":"entered_by_user"},"664152bb1a151874f332b55ec47bc54d":{"name":"Ind.Final Confirmation","dbkey":"ind_final_confirmation"},"196c6ed505a0dabe34f0aed4e2800535":{"name":"Milestone confirmed","dbkey":"milestone_confirmed"},"0b9bf2967b86c25513ff839bcf3eab83":{"name":"Ind. delete doc.","dbkey":"ind_delete_doc"},"attr_data":{"9518cf9f761a3441ba3fe75395cf364d":{"name":"Rf Data","dbkey":"rf_data"},"5e55ab86da1ba43504940280f4877f6e":{"name":"New Data","dbkey":"new_data"},"c7d681312342eb41005e286d676ee1ad":{"name":"other data","dbkey":"other_data"}}}}
\ No newline at end of file
2019-12-31/scott/index.js
View file @ 4372cebd
var fs = require('fs')
var book = require('topsin.excelxs')
var fs = require('fs');
var book = require('topsin.excelxs');
var db = require('topsin.database');
var crypto = require('topsin.crypto')
var crypto = require('topsin.crypto');
// 读取配置文件
var config = JSON.parse(fs.readFile('./config.json'))
var config = JSON.parse(fs.readFile('./config.json'));
// 主
// 主函数
function Main(props){
// 执行初始化
this.init()
this.init();
}
// 初始化方法
Main.prototype.init = function(){
console.log('init')
this.dirlist = this.readDir(config.dirPath, -1) // 获取文件夹下的文件列表
if(this.dirlist.length === 0){ // 没有文件就退出
return
}
this.readXlsx(this.dirlist)
this.readXlsx(this.dirlist) // 循环读取文件列表
}
// 读取文件夹 获取xlsx
Main.prototype.readDir = function(path){
......@@ -27,7 +29,7 @@ Main.prototype.readDir = function(path){
Main.prototype.readXlsx = function(list){
var that = this
list.forEach(function(v){
// 读取
// 读取excel文件
that.excelxs(v.dir + '/' + v.name, v.name)
})
}
......@@ -39,33 +41,76 @@ Main.prototype.excelxs = function(path,name){
if(sheet.isNull()){return}
// 获取表字段数量
var keysCount = sheet.getLastCol()
// 获取数据数量
var dataCount = sheet.getLastRow() - 1
// 获取字段数组
this.keys = []
for(var i=0;i<keysCount;i++){
this.keys = [] // 存放所有字段
this.newkeys = [] // （如果有）存放新的字段
this.attrData = config.dataKeys["attr_data"] // 存放 attr_data
for(var i=0;i<keysCount;i++){ // 读取第一行的字段
if(config.dataKeys[crypto.md5(sheet.getCellValue(0, i))]){ // 如果是之前存在的字段 直接放入keys 以md5加密作为中介
this.keys.push(config.dataKeys[crypto.md5(sheet.getCellValue(0, i))].dbkey)
} else { // 读取不到的字段 可能是attr_data下的 也可能是新增的
if(!this.attrData[crypto.md5(sheet.getCellValue(0, i))]){ // 在attr_data下读取 不存在的是新增的
this.newkeys.push(sheet.getCellValue(0, i)) // 给newkeys添加新增的字段
}
}
}
this.keys.push('attr_data') // keys添加上 attr_data
var that = this
if(this.newkeys.length !== 0){ // 如果有新增的字段
this.newkeys.forEach(function(v){
var mdkey = crypto.md5(v) // md5状态
var dbkeyArr = []
var str = ""
var tempv = v + " "
for(var i = 0,len=tempv.length;i<len;i++){ // 数据库字段 命名规则
if(/[a-zA-Z]/.test(tempv[i])){
str += tempv[i].toLowerCase()
} else if(tempv[i] === " " || tempv[i] === "\n"){
dbkeyArr.push(str)
str = ""
} else {
return
}
}
var dbkey = dbkeyArr.join("_")
that.attrData[mdkey] = { // 内存中的attrData添加新字段的map
name:v,
dbkey: dbkey
}
config.dataKeys["attr_data"][mdkey] = { // 配置里也要添加
name:v,
dbkey: dbkey
}
})
}
var datalist = [] // 定义数组 两次循环获取表数据
for(var j = 1,length = sheet.getLastRow();j<length;j++){
datalist[j-1] = {}
datalist[j-1]["attr_data"] = {} // 对attr_data的处理
for(var i = 0;i<keysCount;i++){
if(i ===5 || i===6){
// var formatdate = sheet.getCellFormat(j, i)
datalist[j-1][config.dataKeys[crypto.md5(sheet.getCellValue(0, i))].dbkey] = sheet.getCellValue(j, i)
// dbkey 是 对应到数据库的key
var dbkey = config.dataKeys[crypto.md5(sheet.getCellValue(0, i))] ? config.dataKeys[crypto.md5(sheet.getCellValue(0, i))].dbkey : null
if(dbkey === "posting_date"){ // 对日期的特殊处理
datalist[j-1][dbkey] = formatDate(sheet.getCellValue(j, i))
} else if (dbkey === "confirmation_entry_time") { // 对时间的特殊处理
datalist[j-1][dbkey] = formatTime(sheet.getCellValue(j, i))
} else if(!config.dataKeys[crypto.md5(sheet.getCellValue(0, i))]) { // 对attr_data字段的特殊处理
datalist[j-1]["attr_data"][this.attrData[crypto.md5(sheet.getCellValue(0, i))].dbkey] = sheet.getCellValue(j, i)
} else {
datalist[j-1][config.dataKeys[crypto.md5(sheet.getCellValue(0, i))].dbkey] = sheet.getCellValue(j, i)
datalist[j-1][dbkey] = sheet.getCellValue(j, i)
}
}
}
// 数据处理
this.paytionType({
keysCount:keysCount,
datalist:datalist
})
book.saveAs(config.rdir + '/' + name) // 读取完的文件另存为 然后可以删除
// 删除
// fs.unlink(path)
// 删除处理过的文件
fs.unlink(path)
book.close()
}
}
Main.prototype.paytionType = function(props){ // 处理数据
......@@ -73,31 +118,47 @@ Main.prototype.paytionType = function(props){ // 处理数据
this.tablename = 'sec_production_order_confirmation_2' // 设置数据库表名
this.query = db.query('Scott_PMO_TRAINING'); // 获取操作对象
var dbDatas = getDbData(this.query,this.tablename) // 获取数据库的所有数据
console.log(dbDatas.length);
if(!dbDatas){return}
if(props.keysCount <= Object.keys(config.dataKeys).length){ // 当字段数量小于等于配置文件字段数量时候 只新增
var res = filterOldData(props.datalist,dbDatas) // 数据过滤
var newData = res.newData
this.insertAny(newData)
} else {
console.log('要对字段扩展')
// this.UpdateKeys()
console.log("数据库的数据数量：" + dbDatas.length)
if(this.newkeys.length === 0){ // 不存在新的字段 只要新增就行
console.log("插入操作");
var res = this.filterOldData(props.datalist,dbDatas) // 数据过滤
var newData = res.newData // 获取要插入的数据
console.log("新的数据："+newData.length);
console.log(newData[0]);
// console.log(this.query.deleteRow({
// table:'sec_production_order_confirmation_2',
// where: ["id >= 0"]
// })) // 删除所有数据
if(newData.length === 0) {return}
this.insertAny(newData) // 插入数据库操作
} else { // 对旧的数据执行更新操作 对新的数据执行添加
console.log('要对字段扩展:')
var res = this.filterOldData(props.datalist,dbDatas,"update") // 数据过滤 顺便更新旧的数据
var newData = res.newData // 获取要插入的数据
console.log("新的数据："+newData.length);
console.log(newData[0]);
// 插入新数据
if(newData.length === 0) {
return
}
this.insertAny(newData) // 插入数据库操作
}
db.removeConnection('Scott_PMO_TRAINING'); // 移除数据库操作
}
Main.prototype.insertAny = function(datas){ // 批量插入数据
this.query.begin()
var that = this
try{
this.query.batchInsert(
this.tablename,
this.keys,
that.tablename,
that.keys,
datas
);
if (this.query.lastError().isValid()){
throw this.query.lastError().text();
}
// if (this.query.lastError().isValid()){
// throw this.query.lastError().text();
// }
this.query.commit();
}
catch(err) {
......@@ -107,28 +168,30 @@ Main.prototype.insertAny = function(datas){ // 批量插入数据
}
}
function getDbData(query,tablename){ // 获取数据库所有数据
query.begin()
query.begin();
var data;
try {
data = query.selectArrayMap({
table: tablename,
field: '*',
field_format:{"tags":'array', attr_data:'json'}
})
});
if (query.lastError().isValid()){
throw query.lastError().text();
}
query.commit();
} catch (error) {
console.log("获取数据库数据错误")
console.log(error)
console.log("获取数据库数据错误");
console.log(error);
query.rollback();
// data = false;
}
return data
}
// key过滤 根据唯一的key将数据分为两组 一组是新数据 一组是原来的数据
function filterOldData(datas,dbdatas, keys){
Main.prototype.filterOldData = function (datas,dbdatas, type){
var type = type || null
console.log("type:"+ type) // 类型 是否执行字段更新操作
// 获取dbdata的唯一字段列表
dbI = dbdatas.map(function(v){
var tempStr = ""
......@@ -140,32 +203,86 @@ function filterOldData(datas,dbdatas, keys){
for(var i = 0,len=datas.length;i<len;i++){
var v = datas[i]
var dataI = v["order_no"] + v["confirmation"] + v["plant","material"] + v["material_description"] + v["posting_date"] + v["confirmation_entry_time"]
if(dbI.indexOf(dataI) > 0){
if(dbI.indexOf(dataI) >= 0){
// 老数据
oldData.push(v)
if(type === "update"){
this.UpdateKeys({"attr_data":v["attr_data"]},dbdatas[dbI.indexOf(dataI)]["id"]) // 更新
}
oldData.push(dbdatas[dbI.indexOf(dataI)])
} else{
// 新数据
newData.push(v)
}
}
console.log("旧的数据："+oldData.length)
console.log(oldData[oldData.length-1]);
return {
oldData:oldData,
newData:newData
}
}
// 更新数据
Main.prototype.UpdateKeys = function(data,id){
// 执行更新操作
this.query.begin()
var that = this
try {
this.query.updateRow({
table:that.tablename,
data:data,
where:{id:id},
update_policy:{tags:'array_append', attr_data:'json_merge'}
})
// if (this.query.lastError().isValid()){
// throw this.query.lastError().text();
// }
this.query.commit();
} catch (error) {
console.log("更新错误")
console.log(error)
this.query.rollback();
}
}
function configReset(){
config.isUseing = false
function formatDate(numb) {
var time = new Date((numb - 1) * 24 * 3600000 + 1)
time.setYear(time.getFullYear() - 70)
var year = time.getFullYear() + ''
var month = time.getMonth() + 1 + ''
var date = time.getDate() + ''
return year + "-" + (month<10?"0"+month:month) + "-" + (date<10?"0"+date:date)
}
function formatTime(numb) {
var hourTmp = numb * 24;
var hour = Math.floor(hourTmp);
var minuteTmp = hourTmp - hour;
var minute = Math.floor(minuteTmp * 60);
var secondTmp = (minuteTmp*60) - minute;
var second = Math.round(secondTmp*60);
if(second >= 60){
second = 0
minute += 1
}
if(minute >= 60) {
minute = minute - 60
hour += 1
}
if(hour >= 24) {
hour = 0
}
return (hour<10? "0" + hour : hour) + ":" + (minute<10? "0" + minute : minute) + ":" + (second<10? "0" + second : second)
}
function configReset(bol){
config.isUseing = bol
fs.writeFile('./config.json',JSON.stringify(config)) // 覆盖写入json
}
if(!config.isUseing){ // 如果没在运行程序就可以实例化
config.isUseing = true
fs.writeFile('./config.json',JSON.stringify(config)) // 覆盖写入json
configReset(true) // 状态变成运行中
var demo = new Main()
book.close()
configReset() // json变成未使用状态
configReset(false) // 状态变成未运行
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment