设为首页 加入收藏

TOP

Python多线程批量插入数据小结(二)
2018-11-20 22:09:39 】 浏览:255
Tags:Python 线程 批量 插入 数据 小结
der_columns="Fcheck_bank, Facc_day, Fbankaccno, Fbankusername, Famount, Fdc_type_id, Fori_accno, Fbank_status,Fid,Fbank_listid,Fbatchno"


#data
order_base="'4251','2018022721','6225425177777777800004','wltest','1','16','6225425177777777800004','00'"
tablename ="epcc_check_201810.t_dc_list_06"



def writeDownSqlData(fileName,content):
    fo = open(fileName,"w")
    fo.write(content)
    fo.close()


def make_t_tcpay_list(deal_num,Flistid,Ftde_id,Fbank_list):


 
    filename ="data_order_"+Fbank_list+".text"


    Fbatchno = 'B201810070011'
    orderDataList = []


    for i in range(int(deal_num)):
        orderData=""
        orderData = "%s,'%s','%s','%s'" % (order_base, Ftde_id, Fbank_list, Fbatchno)
        orderDataList.append(orderData)
        Ftde_id = int(Ftde_id) + 1
        Flistid = int(Flistid) + 1
        Fbank_list = int(Fbank_list) + 1


    writeDownSqlData(resultDir+filename, "\n".join(orderDataList) + "\n")


    loadDataIntoDB(resultDir+filename,tablename,order_columns)


def loadDataIntoDB(filename,tableName,order_columns):
    mysqlCmd = r"LOAD DATA LOCAL INFILE '%s' into table %s FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\''  LINES TERMINATED BY '\n' (%s"%(filename,tableName,order_columns)+")"


    mysqlConCmd = r"mysql -u%s -p%s -h%s"%(dbUser,dbPasswd,dbHost)
    runCmd = mysqlConCmd+' -e "'+mysqlCmd+'"'
    result = call(runCmd,shell=True)
    if result != 0:
        print "load local data into database failed,exit "
        sys.exit()



if __name__ == "__main__":
    start = time.time()


    begin_Flistid ='110180809100012153304210311120'
    begin_Ftde_id ="1"
    begin_Fbank_seq ="2018100800000110734321790770100"
    total_num  =20000000
    threadpool_num = 20
    func_var=[]
    seperate =2000


    pool = threadpool.ThreadPool(threadpool_num)


    for i  in range(seperate):
        list_temp =[]
        list_temp =[str(total_num/seperate),begin_Flistid,begin_Ftde_id,begin_Fbank_seq]
        func_var.append((list_temp,None))
        begin_Flistid = str(int(begin_Flistid)+ total_num/threadpool_num)
        begin_Ftde_id = str(int(begin_Ftde_id)+ total_num/threadpool_num)
        begin_Fbank_seq = str(int(begin_Fbank_seq)+ total_num/threadpool_num)


    pool = threadpool.ThreadPool(threadpool_num)
    requests = threadpool.makeRequests(make_t_tcpay_list, func_var)
    for req in requests:
        pool.putRequest(req)
    pool.wait()


    end = time.time()
    print end - start


首页 上一页 1 2 下一页 尾页 2/2/2
】【打印繁体】【投稿】【收藏】 【推荐】【举报】【评论】 【关闭】 【返回顶部
上一篇C语言实现掷骰子游戏代码及解析 下一篇HTTPS心得笔记之OpenSSL生成root ..

最新文章

热门文章

Hot 文章

Python

C 语言

C++基础

大数据基础

linux编程基础

C/C++面试题目