s

2019-12-30/.gitignore

+node_modules
+.vscode
+

2019-12-30/new/config.json

+{"dirPath":"../xlsx","isUseing":false,"rdir":"../rxlsx","dbconfig":{"database_type":"pg","database_host":"139.196.104.13:5433","database_name":"PMO_TRAINING","database_user":"toplinker","database_pwd":"TopLinker0510"},"keys":["order_no","confirmation","plant","material","material_description","posting_date","confirmation_entry_time","activity","work_center","confirmed_yield","base_unit","confirmed_scrap","reason_for_variance","rework","confirmation_text","entered_by_user","ind_final_confirmation","milestone_confirmed","ind_delete_doc"]}
\ No newline at end of file

2019-12-30/new/index.js

+var fs = require('fs')
+var book = require('topsin.excelxs')
+var db = require('topsin.database');
+
+// 读取配置文件
+var config = JSON.parse(fs.readFile('./config.json'))
+
+// 主
+function Main(props){
+    this.dirPath = props.dirPath  // xlsx文件夹目录
+    // 执行初始化
+    this.init()
+}
+// 初始化方法
+Main.prototype.init = function(){
+    console.log('init')
+    this.dirlist = this.readDir(this.dirPath, -1)
+    if(this.dirlist.length === 0){  // 没有文件就退出
+        return
+    }
+    this.readXlsx(this.dirlist)
+}
+// 读取文件夹 获取xlsx
+Main.prototype.readDir = function(path){
+    return fs.listDir(path).filter(function(v){return v.completeSuffix === "XLSX"})
+}
+// 循环读取文件列表 
+Main.prototype.readXlsx = function(list){
+    var that = this
+    list.forEach(function(v){
+        // 读取
+        that.excelxs(v.dir + '/' + v.name, v.name)
+    })
+}
+
+// 读取excelxs
+Main.prototype.excelxs = function(path,name){
+    if(book.open(path)) {
+        var sheet = book.getActiveSheet()  // 获取当前表
+        
+        if(sheet.isNull()){return}
+
+        // 获取表字段数量
+        var keysCount = sheet.getLastCol()
+        // 获取数据数量
+        var dataCount = sheet.getLastRow() - 1
+        var datalist = []  // 定义数组 两次循环获取表数据
+        for(var j = 1,length = sheet.getLastRow();j<length;j++){
+            datalist[j-1] = []
+            for(var i = 0;i<keysCount;i++){
+                if(i ===5 || i===6){
+                    var formatdate = sheet.getCellFormat(j, i)
+                    formatdate.setNumFormat('DATE')
+                    datalist[j-1].push(sheet.getCellValue(j, i, formatdate))
+                } else {
+                    datalist[j-1].push(sheet.getCellValue(j, i))
+                }
+            }
+        }
+        this.paytionType({
+            keysCount:keysCount,
+            dataCount:dataCount,
+            datalist:datalist
+        })
+        book.saveAs(config.rdir + '/' + name)  // 读取完的文件另存为 然后可以删除
+        // 删除
+        // fs.unlink(path)
+    }
+}
+Main.prototype.paytionType = function(props){  // 该如何处理数据
+    if(props.keysCount <= config.keys.length){ // 当字段数量小于等于配置文件字段数量时候 只新增
+        this.InsertData(props.datalist, config.keys)
+    } else {
+        console.log('要对字段扩展')
+    }
+}
+
+Main.prototype.InsertData = function (datas, keys) {  // 新增数据
+    console.log("数据库操作")
+    db.addConnection(config.dbconfig, "Scott_PMO_TRAINING");
+    var tablename = 'sec_production_order_confirmation_2'
+    var query = db.query('Scott_PMO_TRAINING');
+    var dbDatas = getDbData(query,tablename)
+    // 遍历给数据添加key
+    datas = datas.map(function(v){
+        var obj = {}
+        v.map(function(v2, i2){
+            obj[keys[i2]] = v2
+        })
+        return {
+            key: v[0]+v[1]+v[2]+v[4]+v[5]+v[6],
+            data: obj
+        }
+    })
+    console.log(datas[0].data)
+    // 插入一条数据
+    // var id = query.insertRow({
+    //     table:tablename,
+    //     data:datas[0].data,
+    //     return_field:'id',
+    // });
+    // console.log(id)
+    db.removeConnection('Scott_PMO_TRAINING');  // 移除数据库操作
+}
+
+function getDbData(query,tablename){
+    return query.selectArrayMap({
+        table:tablename,
+        field: '*',
+        field_format:{"tags":'array', attr_data:'json'}
+    })
+}
+
+if(!config.isUseing){  // 如果没在运行程序就可以实例化
+    config.isUseing = true
+    fs.writeFile('./config.json',JSON.stringify(config)) // 覆盖写入json
+    var demo = new Main(config)
+    
+    book.close()
+    config.isUseing = false
+    fs.writeFile('./config.json',JSON.stringify(config)) // 覆盖写入json
+}
+
+
+
+// ["attr_data","action_data","extra_data","tags","sys_tags","sys_data"]
+

2019-12-30/new/md.md

+## 第一步
+
+1. 访问文件夹，获取文件列表，提取xlsx文件
+2. 读取配置文件config.json
+3. 读取xlsx文件，并将已经读取的文件剪切到rxlsx目录
+4. 循环读取excel表里的数据
+5. 提取数据，先处理新增
+6. 遍历数据，获取唯一键，对应到数据库，如果数据库不存在这个数据，就新增
\ No newline at end of file

2019-12-30/order/config.js

+module.exports = {
+    keys: ['id'],
+    dbconfig: {
+        database_type:'pg',
+        database_host:'139.196.104.13:5433',
+        database_name:'PMO_TRAINING',
+        database_user:'toplinker',
+        database_pwd:'TopLinker0510'
+    }
+}
\ No newline at end of file

2019-12-30/order/index.js

+var fs = require('fs')
+var util = require('./util.js')
+var parser = require('topsin.parser');
+var config = require('./config.js')
+var db = require('topsin.database');
+//  创建数据库连接
+db.addConnection(config.dbconfig, "Scott_PMO_TRAINING");
+// 表名
+var tablename = 'sec_production_order_confirmation_2'
+var query = db.query('Scott_PMO_TRAINING');
+
+function init(){
+    // 获取文件列表 提取出csv后缀的文件
+    var list = util.readDir('../resource','csv')  
+    // 对文件名md5加密 获取加密后的文件名列表
+    var csvs = list.map(function(v){
+        return util.crypto('md5',v.name) 
+    })
+    // 获取temp.json的内容
+    var temp = JSON.parse(fs.readFile('./temp.json'))
+    // 过滤掉已经写入的文件,更新json文件
+    var newcsvs = csvs.filter(function(v){
+        if(temp.csvs.indexOf(v) < 0){
+            // 将它加入json的csvs
+            temp.csvs.push(v)
+            return v
+        }
+    })
+    // 判断是否继续执行
+    var isContinue = false
+    var type = []
+    if (config.keys.length !== temp.keys.length){
+        type.push(0)
+        isContinue = true // 字段增加的情况 要执行更新字段的操作
+        temp.keys = config.keys
+    }
+    fs.writeFile('./temp.json',JSON.stringify(temp)) // 覆盖写入json
+    // 判断是否有新增的文件 或者字段是不是没增加 没有则
+    if(newcsvs.length !== 0){
+        type.push(1)
+        isContinue = true
+    }
+    if(isContinue){
+        main({
+            list: list,  // 所有的文件
+            newcsvs: newcsvs,  // 本次新增的文件
+            type: type  // 操作类型(更新字段，新增数据，同时进行)
+        })
+    }
+}
+
+function main(props){
+    var sqldata = SqlDatas()
+    if(props.type.length === 1 && props.type[0] === 0){
+        console.log('只要更新字段')
+    }else{
+        // 获取需要加密的文件
+        var list = props.list.filter(function(v){
+            return props.newcsvs.indexOf(util.crypto('md5',v.name)) >= 0
+        })
+        var datas = []
+        list.forEach(function(v){
+            // 合并文件的所有数据
+            datas = datas.concat(getCsvData(v.dir + '/' + v.name)) 
+        });
+        if(props.type.length === 1){
+            console.log('新增数据')
+            InsertData(datas,sqldata)
+        }else{
+            console.log("更新字段和新增")
+        }
+    }
+    
+}
+
+
+function SqlDatas(){
+    return query.selectArrayMap({
+        table:tablename,
+        field: '*',
+        field_format:{"tags":'array', attr_data:'json'}
+    });
+}
+
+function InsertData(datas,sqldata){
+    
+    // console.log(sqldata)
+    // console.log(datas)
+    console.log(query.getFieldList(tablename))
+}
+
+
+
+
+
+
+
+
+function getCsvData(path){
+    var csvdata = parser.parseCsvStr(fs.readFile(path));   // 获取csv数组
+    csvdata.shift();  // 除去第一行
+    return csvdata
+}
+
+
+
+
+init()
+
+
+db.removeConnection('Scott_PMO_TRAINING');
\ No newline at end of file

2019-12-30/order/temp.json

+{"csvs":["2e2490e8cf4bae28db795c66efeff52d"],"keys":["id"]}
\ No newline at end of file

2019-12-30/order/util.js

+var fs = require('fs')
+var crypto = require('topsin.crypto');
+module.exports = {
+    readDir:function(path, suffix, level){
+        var level = level || -1
+        return fs.listDir(path, level).filter(function(v){
+            return v.completeSuffix === suffix
+        })
+    },
+    crypto: function(type,str){
+        return crypto[type](str)
+    }
+}
\ No newline at end of file

2019-12-31/.gitignore

+node_modules
+.vscode
+

2019-12-31/scott/config.json

+{"dirPath":"../xlsx","isUseing":false,"rdir":"../rxlsx","dbconfig":{"database_type":"pg","database_host":"139.196.104.13:5433","database_name":"PMO_TRAINING","database_user":"toplinker","database_pwd":"TopLinker0510"},"keys":["order_no","confirmation","plant","material","material_description","posting_date","confirmation_entry_time","activity","work_center","confirmed_yield","base_unit","confirmed_scrap","reason_for_variance","rework","confirmation_text","entered_by_user","ind_final_confirmation","milestone_confirmed","ind_delete_doc"],"dataKeys":{"135aa36437beb6571531ca8e898ad57a":{"name":"Order\n生管投料单号","dbkey":"order_no"},"a1deefe94c73d05d2a3b247c23362042":{"name":"Confirmation\n总数量","dbkey":"confirmation"},"c46b4eb1d67a08b863072b33b646a810":{"name":"Plant\n厂别","dbkey":"plant"},"e5018abee57b7f610ef8bb4d9eecc112":{"name":"Material\n批号","dbkey":"material"},"f19e926ffab820b96d5a6aabf9aae616":{"name":"Material Description\n批次说明","dbkey":"material_description"},"7e51e12322207097a84891f9d7445ad4":{"name":"Posting Date\n生产时间","dbkey":"posting_date"},"8dba6faf3484a91f3b3940bb2e9ea459":{"name":"Confirmation Entry Time\n确认时间","dbkey":"confirmation_entry_time"},"ecfc2dffe568c10a67dbc6d3724cfde2":{"name":"Activity","dbkey":"activity"},"657a1d49f3e92290e56b692ff58a0a2d":{"name":"Work Center\n生产站别","dbkey":"work_center"},"62f5f57f24ea1d0bc12aa535d6add05d":{"name":"Confirmed Yield (GMEIN)","dbkey":"confirmed_yield"},"7c8590875b72ce8eca3c92f76b2c451b":{"name":"Base Unit of Measure (=GMEIN)","dbkey":"base_unit"},"37c9b123cc9f488d3f7f8480146795e9":{"name":"Confirmed scrap (MEINH)\n报废数量","dbkey":"confirmed_scrap"},"91b8b2e603be16c0a94658f0ced49c0b":{"name":"Reason for Variance\n报废代码","dbkey":"reason_for_variance"},"55979da7be6e74a5ce14bd81cd31028d":{"name":"Rework\n重工数量","dbkey":"rework"},"e820bdb5f699f28fb15c5d5e759f164b":{"name":"Confirmation text","dbkey":"confirmation_text"},"da01091b1f5ddb363d498e844449a961":{"name":"Entered by User","dbkey":"entered_by_user"},"664152bb1a151874f332b55ec47bc54d":{"name":"Ind.Final Confirmation","dbkey":"ind_final_confirmation"},"196c6ed505a0dabe34f0aed4e2800535":{"name":"Milestone confirmed","dbkey":"milestone_confirmed"},"0b9bf2967b86c25513ff839bcf3eab83":{"name":"Ind. delete doc.","dbkey":"ind_delete_doc"}}}
\ No newline at end of file

2019-12-31/scott/index.js

+var fs = require('fs')
+var book = require('topsin.excelxs')
+var db = require('topsin.database');
+var crypto = require('topsin.crypto')
+// 读取配置文件
+var config = JSON.parse(fs.readFile('./config.json'))
+
+// 主
+function Main(props){
+    // 执行初始化
+    this.init()
+}
+// 初始化方法
+Main.prototype.init = function(){
+    console.log('init')
+    this.dirlist = this.readDir(config.dirPath, -1)  // 获取文件夹下的文件列表
+    if(this.dirlist.length === 0){  // 没有文件就退出
+        return
+    }
+    this.readXlsx(this.dirlist)
+}
+// 读取文件夹 获取xlsx
+Main.prototype.readDir = function(path){
+    return fs.listDir(path).filter(function(v){return v.completeSuffix === "XLSX"})
+}
+// 循环读取文件列表 
+Main.prototype.readXlsx = function(list){
+    var that = this
+    list.forEach(function(v){
+        // 读取
+        that.excelxs(v.dir + '/' + v.name, v.name)
+    })
+}
+
+// 读取excelxs
+Main.prototype.excelxs = function(path,name){
+    if(book.open(path)) {
+        var sheet = book.getActiveSheet()  // 获取当前表
+        if(sheet.isNull()){return}
+        // 获取表字段数量
+        var keysCount = sheet.getLastCol()
+        // 获取数据数量
+        var dataCount = sheet.getLastRow() - 1
+        // 获取字段数组
+        this.keys = []
+        for(var i=0;i<keysCount;i++){
+            this.keys.push(config.dataKeys[crypto.md5(sheet.getCellValue(0, i))].dbkey)
+        }
+        var datalist = []  // 定义数组 两次循环获取表数据
+        for(var j = 1,length = sheet.getLastRow();j<length;j++){
+            datalist[j-1] = {}
+            for(var i = 0;i<keysCount;i++){
+                if(i ===5 || i===6){
+                    // var formatdate = sheet.getCellFormat(j, i)
+                    datalist[j-1][config.dataKeys[crypto.md5(sheet.getCellValue(0, i))].dbkey] = sheet.getCellValue(j, i)
+                } else {
+                    datalist[j-1][config.dataKeys[crypto.md5(sheet.getCellValue(0, i))].dbkey] = sheet.getCellValue(j, i)
+                }
+            }
+        }
+        // 数据处理
+        this.paytionType({
+            keysCount:keysCount,
+            datalist:datalist
+        })
+        book.saveAs(config.rdir + '/' + name)  // 读取完的文件另存为 然后可以删除
+        // 删除
+        // fs.unlink(path)
+    }
+}
+Main.prototype.paytionType = function(props){  // 处理数据
+    db.addConnection(config.dbconfig, "Scott_PMO_TRAINING");  // 连接数据库
+    this.tablename = 'sec_production_order_confirmation_2'  // 设置数据库表名
+    this.query = db.query('Scott_PMO_TRAINING');  // 获取操作对象
+    var dbDatas = getDbData(this.query,this.tablename)  // 获取数据库的所有数据
+    console.log(dbDatas.length);
+    
+    if(!dbDatas){return}
+    if(props.keysCount <= Object.keys(config.dataKeys).length){ // 当字段数量小于等于配置文件字段数量时候 只新增
+        var res = filterOldData(props.datalist,dbDatas)  // 数据过滤
+        var newData = res.newData
+        this.insertAny(newData)
+    } else {
+        console.log('要对字段扩展')
+        // this.UpdateKeys()
+    }
+
+    db.removeConnection('Scott_PMO_TRAINING');  // 移除数据库操作
+}
+Main.prototype.insertAny = function(datas){ // 批量插入数据
+    this.query.begin()
+    try{
+        this.query.batchInsert(
+            this.tablename,
+            this.keys,
+            datas
+        );
+        if (this.query.lastError().isValid()){
+            throw this.query.lastError().text();
+        }
+        this.query.commit();
+    }
+    catch(err) {
+        console.log("插入数据错误")
+        console.log(err)
+        this.query.rollback();
+    }
+}
+function getDbData(query,tablename){   // 获取数据库所有数据
+    query.begin()
+    var data;
+    try {
+        data = query.selectArrayMap({
+            table: tablename,
+            field: '*',
+            field_format:{"tags":'array', attr_data:'json'}
+        })
+        if (query.lastError().isValid()){
+            throw query.lastError().text();
+        }
+        query.commit();
+    } catch (error) {
+        console.log("获取数据库数据错误")
+        console.log(error)
+        query.rollback();
+        // data = false;
+    }
+    return data
+}
+// key过滤  根据唯一的key将数据分为两组 一组是新数据 一组是原来的数据
+function filterOldData(datas,dbdatas, keys){
+    // 获取dbdata的唯一字段列表
+    dbI = dbdatas.map(function(v){
+        var tempStr = ""
+        tempStr += v["order_no"] + v["confirmation"] + v["plant","material"] + v["material_description"] + v["posting_date"] + v["confirmation_entry_time"]
+        return tempStr
+    })
+    var oldData = [],
+        newData = [];
+    for(var i = 0,len=datas.length;i<len;i++){
+        var v = datas[i]
+        var dataI = v["order_no"] + v["confirmation"] + v["plant","material"] + v["material_description"] + v["posting_date"] + v["confirmation_entry_time"]
+        if(dbI.indexOf(dataI) > 0){
+            // 老数据
+            oldData.push(v)
+        } else{
+            // 新数据
+            newData.push(v)
+        }
+    }
+    return {
+        oldData:oldData,
+        newData:newData
+    }
+}
+
+function configReset(){
+    config.isUseing = false
+    fs.writeFile('./config.json',JSON.stringify(config)) // 覆盖写入json
+}
+
+if(!config.isUseing){  // 如果没在运行程序就可以实例化
+    config.isUseing = true
+    fs.writeFile('./config.json',JSON.stringify(config)) // 覆盖写入json
+    var demo = new Main()
+    
+    book.close()
+    configReset()  // json变成未使用状态
+}
+
+
+
+// ["attr_data","action_data","extra_data","tags","sys_tags","sys_data"]
+

README.md

@@ -37,3 +37,7 @@
 
 * topsin.database topsin.sqlquery  数据库操作
 * 需求：读取excel数据 存入数据库
+
+## 2019-12-30
+
+*